Merge pull request #2115 from cisagov/dk/1989-export-import-tables

Issue #1989 : export import tables

docs/operations/import_export.md

@@ -0,0 +1,66 @@
+# Export / Import Tables
+
+A means is provided to export and import individual tables from
+one environment to another. This allows for replication of
+production data in a development environment. Import and export
+are provided through the django admin interface, through a modified
+library, django-import-export. Each supported model has an Import
+and an Export button on the list view.
+
+### Export
+
+When exporting models from the source environment, make sure that
+no filters are selected. This will ensure that all rows of the model
+are exported. Due to database dependencies, the following models 
+need to be exported:
+
+* User
+* Contact
+* Domain
+* DomainRequest
+* DomainInformation
+* DomainUserRole
+* DraftDomain
+* Websites
+* Host
+* HostIP
+
+### Import
+
+When importing into the target environment, if the target environment
+is different than the source environment, it must be prepared for the
+import. This involves clearing out rows in the appropriate tables so
+that there are no database conflicts on import.
+
+#### Preparing Target Environment
+
+Delete all rows from tables in the following order through django admin:
+
+* DomainInformation
+* DomainRequest
+* Domain
+* User (all but the current user)
+* Contact
+* Websites
+* DraftDomain
+* HostIP
+* Host
+
+#### Importing into Target Environment
+
+Once target environment is prepared, files can be imported in the following
+order:
+
+* User (After importing User table, you need to delete all rows from Contact table before importing Contacts)
+* Contact
+* Domain
+* Host
+* HostIP
+* DraftDomain
+* Websites
+* DomainRequest
+* DomainInformation
+* UserDomainRole
+
+Optional step:
+* Run fixtures to load fixture users back in

src/Pipfile

@@ -32,6 +32,7 @@ fred-epplib = {git = "https://github.com/cisagov/epplib.git", ref = "master"}
 pyzipper="*"
 tblib = "*"
 django-admin-multiple-choice-list-filter = "*"
+django-import-export = "*"
 django-waffle = "*"
 
 [dev-packages]

src/Pipfile.lock

@@ -272,6 +272,14 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==0.7.1"
         },
+        "diff-match-patch": {
+            "hashes": [
+                "sha256:953019cdb9c9d2c9e47b5b12bcff3cf4746fc4598eb406076fa1fc27e6a1f15c",
+                "sha256:dce43505fb7b1b317de7195579388df0746d90db07015ed47a85e5e44930ef93"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==20230430"
+        },
         "dj-database-url": {
             "hashes": [
                 "sha256:04bc34b248d4c21aaa13e4ab419ae6575ef5f10f3df735ce7da97722caa356e0",
@@ -352,6 +360,15 @@
             "index": "pypi",
             "version": "==2.8.1"
         },
+        "django-import-export": {
+            "hashes": [
+                "sha256:2eac09e8cec8670f36e24314760448011ad23c51e8fb930d55f50d0c3c926da0",
+                "sha256:4deabc557801d368093608c86fd0f4831bc9540e2ea41ca2f023e2efb3eb6f48"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==3.3.8"
+        },
         "django-login-required-middleware": {
             "hashes": [
                 "sha256:847ae9a69fd7a07618ed53192b3c06946af70a0caf6d0f4eb40a8f37593cd970"
@@ -399,6 +416,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==11.0.0"
         },
+        "et-xmlfile": {
+            "hashes": [
+                "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c",
+                "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.1.0"
+        },
         "faker": {
             "hashes": [
                 "sha256:87ef41e24b39a5be66ecd874af86f77eebd26782a2681200e86c5326340a95d3",
@@ -733,6 +758,12 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.3.3"
         },
+        "markuppy": {
+            "hashes": [
+                "sha256:1adee2c0a542af378fe84548ff6f6b0168f3cb7f426b46961038a2bcfaad0d5f"
+            ],
+            "version": "==1.14"
+        },
         "markupsafe": {
             "hashes": [
                 "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf",
@@ -807,6 +838,13 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.21.1"
         },
+        "odfpy": {
+            "hashes": [
+                "sha256:db766a6e59c5103212f3cc92ec8dd50a0f3a02790233ed0b52148b70d3c438ec",
+                "sha256:fc3b8d1bc098eba4a0fda865a76d9d1e577c4ceec771426bcb169a82c5e9dfe0"
+            ],
+            "version": "==1.4.1"
+        },
         "oic": {
             "hashes": [
                 "sha256:b74bd06c7de1ab4f8e798f714062e6a68f68ad9cdbed1f1c30a7fb887602f321",
@@ -816,6 +854,13 @@
             "markers": "python_version ~= '3.8'",
             "version": "==1.7.0"
         },
+        "openpyxl": {
+            "hashes": [
+                "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184",
+                "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"
+            ],
+            "version": "==3.1.2"
+        },
         "orderedmultidict": {
             "hashes": [
                 "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad",
@@ -1088,6 +1133,62 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.0.1"
         },
+        "pyyaml": {
+            "hashes": [
+                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
+                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
+                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
+                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
+                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
+                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
+                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
+                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
+                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
+                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
+                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
+                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
+                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
+                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
+                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
+                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
+                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
+                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
+                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
+                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
+                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
+                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
+                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
+                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
+                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
+                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
+                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
+                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
+                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
+                "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
+                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
+                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
+                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
+                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
+                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
+                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
+                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
+                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
+                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
+                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
+                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
+                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
+                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
+                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
+                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
+                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
+                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
+                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
+                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
+                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
+                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
+            ],
+            "version": "==6.0.1"
+        },
         "pyzipper": {
             "hashes": [
                 "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc",
@@ -1138,6 +1239,21 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.5.0"
         },
+        "tablib": {
+            "extras": [
+                "html",
+                "ods",
+                "xls",
+                "xlsx",
+                "yaml"
+            ],
+            "hashes": [
+                "sha256:9821caa9eca6062ff7299fa645e737aecff982e6b2b42046928a6413c8dabfd9",
+                "sha256:f6661dfc45e1d4f51fa8a6239f9c8349380859a5bfaa73280645f046d6c96e33"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.5.0"
+        },
         "tblib": {
             "hashes": [
                 "sha256:80a6c77e59b55e83911e1e607c649836a69c103963c5f28a46cbeef44acf8129",
@@ -1173,6 +1289,20 @@
             "markers": "python_version >= '3.8'",
             "version": "==6.6.0"
         },
+        "xlrd": {
+            "hashes": [
+                "sha256:6a33ee89877bd9abc1158129f6e94be74e2679636b8a205b43b85206c3f0bbdd",
+                "sha256:f72f148f54442c6b056bf931dbc34f986fd0c3b0b6b5a58d013c9aef274d0c88"
+            ],
+            "version": "==2.0.1"
+        },
+        "xlwt": {
+            "hashes": [
+                "sha256:a082260524678ba48a297d922cc385f58278b8aa68741596a87de01a9c628b2e",
+                "sha256:c59912717a9b28f1a3c2a98fd60741014b06b043936dcecbc113eaaada156c88"
+            ],
+            "version": "==1.3.0"
+        },
         "zope.event": {
             "hashes": [
                 "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26",
@@ -1597,7 +1727,6 @@
                 "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
                 "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
             ],
-            "markers": "python_version >= '3.6'",
             "version": "==6.0.1"
         },
         "rich": {

src/registrar/admin.py

@@ -7,7 +7,7 @@ from django.db.models import Value, CharField, Q
 from django.db.models.functions import Concat, Coalesce
 from django.http import HttpResponseRedirect
 from django.shortcuts import redirect
-from django_fsm import get_available_FIELD_transitions
+from django_fsm import get_available_FIELD_transitions, FSMField
 from django.contrib import admin, messages
 from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
 from django.contrib.auth.models import Group
@@ -30,12 +30,63 @@ from django.utils.safestring import mark_safe
 from django.utils.html import escape
 from django.contrib.auth.forms import UserChangeForm, UsernameField
 from django_admin_multiple_choice_list_filter.list_filters import MultipleChoiceListFilter
+from import_export import resources
+from import_export.admin import ImportExportModelAdmin
 
 from django.utils.translation import gettext_lazy as _
 
 logger = logging.getLogger(__name__)
 
 
+class FsmModelResource(resources.ModelResource):
+    """ModelResource is extended to support importing of tables which
+    have FSMFields.  ModelResource is extended with the following changes
+    to existing behavior:
+    When new objects are to be imported, FSMFields are initialized before
+    the object is initialized.  This is because FSMFields do not allow
+    direct modification.
+    When objects, which are to be imported, are updated, the FSMFields
+    are skipped."""
+
+    def init_instance(self, row=None):
+        """Overrides the init_instance method of ModelResource.  Returns
+        an instance of the model, with the FSMFields already initialized
+        from data in the row."""
+
+        # Get fields which are fsm fields
+        fsm_fields = {}
+
+        for f in self._meta.model._meta.fields:
+            if isinstance(f, FSMField):
+                if row and f.name in row:
+                    fsm_fields[f.name] = row[f.name]
+
+        # Initialize model instance with fsm_fields
+        return self._meta.model(**fsm_fields)
+
+    def import_field(self, field, obj, data, is_m2m=False, **kwargs):
+        """Overrides the import_field method of ModelResource.  If the
+        field being imported is an FSMField, it is not imported."""
+
+        is_fsm = False
+
+        # check each field in the object
+        for f in obj._meta.fields:
+            # if the field is an instance of FSMField
+            if field.attribute == f.name and isinstance(f, FSMField):
+                is_fsm = True
+        if not is_fsm:
+            super().import_field(field, obj, data, is_m2m, **kwargs)
+
+
+class UserResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.User
+
+
 class MyUserAdminForm(UserChangeForm):
     """This form utilizes the custom widget for its class's ManyToMany UIs.
 
@@ -498,9 +549,11 @@ class UserContactInline(admin.StackedInline):
     model = models.Contact
 
 
-class MyUserAdmin(BaseUserAdmin):
+class MyUserAdmin(BaseUserAdmin, ImportExportModelAdmin):
     """Custom user admin class to use our inlines."""
 
+    resource_classes = [UserResource]
+
     form = MyUserAdminForm
 
     class Meta:
@@ -706,17 +759,52 @@ class HostIPInline(admin.StackedInline):
     model = models.HostIP
 
 
-class MyHostAdmin(AuditedAdmin):
+class HostResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.Host
+
+
+class MyHostAdmin(AuditedAdmin, ImportExportModelAdmin):
     """Custom host admin class to use our inlines."""
 
+    resource_classes = [HostResource]
+
     search_fields = ["name", "domain__name"]
     search_help_text = "Search by domain or host name."
     inlines = [HostIPInline]
 
 
-class ContactAdmin(ListHeaderAdmin):
+class HostIpResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.HostIP
+
+
+class HostIpAdmin(AuditedAdmin, ImportExportModelAdmin):
+    """Custom host ip admin class"""
+
+    resource_classes = [HostIpResource]
+    model = models.HostIP
+
+
+class ContactResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.Contact
+
+
+class ContactAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom contact admin class to add search."""
 
+    resource_classes = [ContactResource]
+
     search_fields = ["email", "first_name", "last_name"]
     search_help_text = "Search by first name, last name or email."
     list_display = [
@@ -837,9 +925,19 @@ class ContactAdmin(ListHeaderAdmin):
         return super().change_view(request, object_id, form_url, extra_context=extra_context)
 
 
-class WebsiteAdmin(ListHeaderAdmin):
+class WebsiteResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.Website
+
+
+class WebsiteAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom website admin class."""
 
+    resource_classes = [WebsiteResource]
+
     # Search
     search_fields = [
         "website",
@@ -887,9 +985,19 @@ class WebsiteAdmin(ListHeaderAdmin):
         return response
 
 
-class UserDomainRoleAdmin(ListHeaderAdmin):
+class UserDomainRoleResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.UserDomainRole
+
+
+class UserDomainRoleAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom user domain role admin class."""
 
+    resource_classes = [UserDomainRoleResource]
+
     class Meta:
         """Contains meta information about this class"""
 
@@ -970,9 +1078,19 @@ class DomainInvitationAdmin(ListHeaderAdmin):
     change_form_template = "django/admin/email_clipboard_change_form.html"
 
 
-class DomainInformationAdmin(ListHeaderAdmin):
+class DomainInformationResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.DomainInformation
+
+
+class DomainInformationAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Customize domain information admin class."""
 
+    resource_classes = [DomainInformationResource]
+
     form = DomainInformationAdminForm
 
     # Columns
@@ -1101,9 +1219,19 @@ class DomainInformationAdmin(ListHeaderAdmin):
         return readonly_fields  # Read-only fields for analysts
 
 
-class DomainRequestAdmin(ListHeaderAdmin):
+class DomainRequestResource(FsmModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.DomainRequest
+
+
+class DomainRequestAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom domain requests admin class."""
 
+    resource_classes = [DomainRequestResource]
+
     form = DomainRequestAdminForm
     change_form_template = "django/admin/domain_request_change_form.html"
 
@@ -1644,9 +1772,19 @@ class DomainInformationInline(admin.StackedInline):
         return DomainInformationAdmin.get_readonly_fields(self, request, obj=None)
 
 
-class DomainAdmin(ListHeaderAdmin):
+class DomainResource(FsmModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.Domain
+
+
+class DomainAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom domain admin class to add extra buttons."""
 
+    resource_classes = [DomainResource]
+
     class ElectionOfficeFilter(admin.SimpleListFilter):
         """Define a custom filter for is_election_board"""
 
@@ -2041,9 +2179,19 @@ class DomainAdmin(ListHeaderAdmin):
         return super().has_change_permission(request, obj)
 
 
-class DraftDomainAdmin(ListHeaderAdmin):
+class DraftDomainResource(resources.ModelResource):
+    """defines how each field in the referenced model should be mapped to the corresponding fields in the
+    import/export file"""
+
+    class Meta:
+        model = models.DraftDomain
+
+
+class DraftDomainAdmin(ListHeaderAdmin, ImportExportModelAdmin):
     """Custom draft domain admin class."""
 
+    resource_classes = [DraftDomainResource]
+
     search_fields = ["name"]
     search_help_text = "Search by draft domain name."
 
@@ -2179,9 +2327,8 @@ admin.site.register(models.DomainInformation, DomainInformationAdmin)
 admin.site.register(models.Domain, DomainAdmin)
 admin.site.register(models.DraftDomain, DraftDomainAdmin)
 admin.site.register(models.FederalAgency, FederalAgencyAdmin)
-# Host and HostIP removed from django admin because changes in admin
-# do not propagate to registry and logic not applied
 admin.site.register(models.Host, MyHostAdmin)
+admin.site.register(models.HostIP, HostIpAdmin)
 admin.site.register(models.Website, WebsiteAdmin)
 admin.site.register(models.PublicContact, PublicContactAdmin)
 admin.site.register(models.DomainRequest, DomainRequestAdmin)

src/registrar/assets/sass/_theme/_admin.scss

@@ -717,6 +717,10 @@ div.dja__model-description{
 
 }
 
+.import_export_text {
+    color: var(--secondary);
+}
+
 .text-underline {
     text-decoration: underline !important;
 }

src/registrar/config/settings.py

@@ -147,6 +147,8 @@ INSTALLED_APPS = [
     "corsheaders",
     # library for multiple choice filters in django admin
     "django_admin_multiple_choice_list_filter",
+    # library for export and import of data
+    "import_export",
     # Waffle feature flags
     "waffle",
 ]

src/registrar/models/utility/generic_helper.py

@@ -100,36 +100,41 @@ class CreateOrUpdateOrganizationTypeHelper:
         # Update the field
         self._update_fields(organization_type_needs_update, generic_org_type_needs_update)
 
-    def _handle_existing_instance(self, force_update_when_no_are_changes_found=False):
+    def _handle_existing_instance(self, force_update_when_no_changes_are_found=False):
         # == Init variables == #
-        # Instance is already in the database, fetch its current state
+        try:
-        current_instance = self.sender.objects.get(id=self.instance.id)
+            # Instance is already in the database, fetch its current state
+            current_instance = self.sender.objects.get(id=self.instance.id)
 
-        # Check the new and old values
+            # Check the new and old values
-        generic_org_type_changed = self.instance.generic_org_type != current_instance.generic_org_type
+            generic_org_type_changed = self.instance.generic_org_type != current_instance.generic_org_type
-        is_election_board_changed = self.instance.is_election_board != current_instance.is_election_board
+            is_election_board_changed = self.instance.is_election_board != current_instance.is_election_board
-        organization_type_changed = self.instance.organization_type != current_instance.organization_type
+            organization_type_changed = self.instance.organization_type != current_instance.organization_type
 
-        # == Check for invalid conditions before proceeding == #
+            # == Check for invalid conditions before proceeding == #
-        if organization_type_changed and (generic_org_type_changed or is_election_board_changed):
+            if organization_type_changed and (generic_org_type_changed or is_election_board_changed):
-            # Since organization type is linked with generic_org_type and election board,
+                # Since organization type is linked with generic_org_type and election board,
-            # we have to update one or the other, not both.
+                # we have to update one or the other, not both.
-            # This will not happen in normal flow as it is not possible otherwise.
+                # This will not happen in normal flow as it is not possible otherwise.
-            raise ValueError("Cannot update organization_type and generic_org_type simultaneously.")
+                raise ValueError("Cannot update organization_type and generic_org_type simultaneously.")
-        elif not organization_type_changed and (not generic_org_type_changed and not is_election_board_changed):
+            elif not organization_type_changed and (not generic_org_type_changed and not is_election_board_changed):
-            # No changes found
+                # No changes found
-            if force_update_when_no_are_changes_found:
+                if force_update_when_no_changes_are_found:
-                # If we want to force an update anyway, we can treat this record like
+                    # If we want to force an update anyway, we can treat this record like
-                # its a new one in that we check for "None" values rather than changes.
+                    # its a new one in that we check for "None" values rather than changes.
-                self._handle_new_instance()
+                    self._handle_new_instance()
-        else:
+            else:
-            # == Update the linked values == #
+                # == Update the linked values == #
-            # Find out which field needs updating
+                # Find out which field needs updating
-            organization_type_needs_update = generic_org_type_changed or is_election_board_changed
+                organization_type_needs_update = generic_org_type_changed or is_election_board_changed
-            generic_org_type_needs_update = organization_type_changed
+                generic_org_type_needs_update = organization_type_changed
 
-            # Update the field
+                # Update the field
-            self._update_fields(organization_type_needs_update, generic_org_type_needs_update)
+                self._update_fields(organization_type_needs_update, generic_org_type_needs_update)
+        except self.sender.DoesNotExist:
+            # this exception should only be raised when import_export utility attempts to import
+            # a new row and already has an id
+            pass
 
     def _update_fields(self, organization_type_needs_update, generic_org_type_needs_update):
         """

src/registrar/templates/admin/import_export/change_list_import_item.html

@@ -0,0 +1,8 @@
+{% load i18n %}
+{% load admin_urls %}
+
+{% if has_import_permission %}
+  {% if not IS_PRODUCTION %}
+<li><a href='{% url opts|admin_urlname:"import" %}' class="import_link">{% trans "Import" %}</a></li>
+  {% endif %}
+{% endif %}

src/registrar/templates/admin/import_export/import.html

@@ -0,0 +1,191 @@
+{% extends "admin/import_export/base.html" %}
+{% load i18n %}
+{% load admin_urls %}
+{% load import_export_tags %}
+{% load static %}
+
+{% block extrastyle %}{{ block.super }}<link rel="stylesheet" type="text/css" href="{% static "import_export/import.css" %}" />{% endblock %}
+
+{% block extrahead %}{{ block.super }}
+  <script type="text/javascript" src="{% url 'admin:jsi18n' %}"></script>
+  {% if confirm_form %}
+    {{ confirm_form.media }}
+  {% else %}
+    {{ form.media }}
+  {% endif %}
+{% endblock %}
+
+{% block breadcrumbs_last %}
+{% trans "Import" %}
+{% endblock %}
+
+{% block content %}
+
+  {% if confirm_form %}
+    {% block confirm_import_form %}
+    <form action="{% url opts|admin_urlname:"process_import" %}" method="POST">
+      {% csrf_token %}
+      {{ confirm_form.as_p }}
+      <p class="import_export_text">
+        {% trans "Below is a preview of data to be imported. If you are satisfied with the results, click 'Confirm import'" %}
+      </p>
+      <div class="submit-row">
+        <input type="submit" class="default" name="confirm" value="{% trans "Confirm import" %}">
+      </div>
+    </form>
+    {% endblock %}
+  {% else %}
+    {% block import_form %}
+    <form action="" method="post" enctype="multipart/form-data">
+      {% csrf_token %}
+
+      {% include "admin/import_export/resource_fields_list.html" with import_or_export="import" %}
+      {% block import_form_additional_info %}{% endblock %}
+
+      {% block form_detail %}
+          <fieldset class="module aligned">
+          {% for field in form %}
+            <div class="form-row">
+              {{ field.errors }}
+
+              {{ field.label_tag }}
+
+              {{ field }}
+
+              {% if field.field.help_text %}
+              <p class="help">{{ field.field.help_text|safe }}</p>
+              {% endif %}
+            </div>
+          {% endfor %}
+        </fieldset>
+      {% endblock %}
+
+      {% block form_submit_button %}
+        <div class="submit-row">
+          <input type="submit" class="default" value="{% trans "Submit" %}">
+        </div>
+      {% endblock %}
+    </form>
+    {% endblock %}
+  {% endif %}
+
+  {% if result %}
+
+    {% if result.has_errors %}
+    {% block errors %}
+      <h2>{% trans "Errors" %}</h2>
+      <ul>
+        {% for error in result.base_errors  %}
+        <li class="import_export_text">
+          {{ error.error }}
+          <div class="traceback">{{ error.traceback|linebreaks }}</div>
+        </li>
+        {% endfor %}
+        {% for line, errors in result.row_errors %}
+          {% for error in errors %}
+            <li class="import_export_text">
+              {% trans "Line number" %}: {{ line }} - {{ error.error }}
+              <div><code class="import_export_text">{{ error.row.values|join:", " }}</code></div>
+              <div class="traceback">{{ error.traceback|linebreaks }}</div>
+            </li>
+          {% endfor %}
+        {% endfor %}
+      </ul>
+    {% endblock %}
+
+    {% elif result.has_validation_errors %}
+
+    {% block validation_errors %}
+      <h2>{% trans "Some rows failed to validate" %}</h2>
+
+      <p>{% trans "Please correct these errors in your data where possible, then reupload it using the form above." %}</p>
+
+      <table class="import-preview">
+        <thead>
+          <tr>
+            <th>{% trans "Row" %}</th>
+            <th>{% trans "Errors" %}</th>
+            {% for field in result.diff_headers %}
+              <th>{{ field }}</th>
+            {% endfor %}
+          </tr>
+        </thead>
+        <tbody>
+        {% for row in result.invalid_rows %}
+          <tr>
+            <td>{{ row.number }} </td>
+            <td class="errors">
+              <span class="validation-error-count">{{ row.error_count }}</span>
+              <div class="validation-error-container">
+                <ul class="validation-error-list">
+                  {% for field_name, error_list in row.field_specific_errors.items %}
+                    <li>
+                        <span class="validation-error-field-label">{{ field_name }}</span>
+                        <ul>
+                          {% for error in error_list %}
+                            <li>{{ error }}</li>
+                          {% endfor %}
+                        </ul>
+                    </li>
+                  {% endfor %}
+                  {% if row.non_field_specific_errors %}
+                    <li>
+                      <span class="validation-error-field-label">{% trans "Non field specific" %}</span>
+                      <ul>
+                        {% for error in row.non_field_specific_errors %}
+                          <li>{{ error }}</li>
+                        {% endfor %}
+                      </ul>
+                    </li>
+                  {% endif %}
+                </ul>
+              </div>
+            </td>
+            {% for field in row.values %}
+              <td>{{ field }}</td>
+            {% endfor %}
+          </tr>
+        {% endfor %}
+        </tbody>
+      </table>
+      {% endblock %}
+
+    {% else %}
+
+      {% block preview %}
+      <h2>{% trans "Preview" %}</h2>
+
+      <table class="import-preview">
+        <thead>
+          <tr>
+            <th></th>
+            {% for field in result.diff_headers %}
+              <th>{{ field }}</th>
+            {% endfor %}
+          </tr>
+        </thead>
+        {% for row in result.valid_rows %}
+          <tr class="{{ row.import_type }}">
+            <td class="import-type">
+              {% if row.import_type == 'new' %}
+                {% trans "New" %}
+              {% elif row.import_type == 'skip' %}
+                {% trans "Skipped" %}
+              {% elif row.import_type == 'delete' %}
+                {% trans "Delete" %}
+              {% elif row.import_type == 'update' %}
+                {% trans "Update" %}
+              {% endif %}
+            </td>
+            {% for field in row.diff %}
+              <td>{{ field }}</td>
+            {% endfor %}
+          </tr>
+        {% endfor %}
+      </table>
+      {% endblock %}
+
+    {% endif %}
+
+  {% endif %}
+{% endblock %}

src/registrar/templates/admin/import_export/resource_fields_list.html

@@ -0,0 +1,21 @@
+{% load i18n %}
+{% block fields_help %}
+  <p class="import_export_text">
+    {% if import_or_export == "export" %}
+      {% trans "This exporter will export the following fields: " %}
+    {% elif import_or_export == "import" %}
+      {% trans "This importer will import the following fields: " %}
+    {% endif %}
+
+    {% if fields_list|length <= 1 %}
+       <code class="import_export_text">{{ fields_list.0.1|join:", " }}</code>
+    {% else %}
+       <dl>
+          {% for resource, fields in fields_list %}
+              <dt>{{ resource }}</dt>
+              <dd><code class="import_export_text">{{ fields|join:", " }}</code></dd>
+          {% endfor %}
+       </dl>
+    {% endif %}
+  </p>
+{% endblock %}

src/registrar/tests/test_admin.py

@@ -21,6 +21,7 @@ from registrar.admin import (
     MyHostAdmin,
     UserDomainRoleAdmin,
     VerifiedByStaffAdmin,
+    FsmModelResource,
     WebsiteAdmin,
     DraftDomainAdmin,
     FederalAgencyAdmin,
@@ -61,7 +62,7 @@ from .common import (
 )
 from django.contrib.sessions.backends.db import SessionStore
 from django.contrib.auth import get_user_model
-from unittest.mock import ANY, call, patch
+from unittest.mock import ANY, call, patch, Mock
 from unittest import skip
 
 from django.conf import settings
@@ -71,6 +72,49 @@ import logging
 logger = logging.getLogger(__name__)
 
 
+class TestFsmModelResource(TestCase):
+    def setUp(self):
+        self.resource = FsmModelResource()
+
+    def test_init_instance(self):
+        """Test initializing an instance of a class with a FSM field"""
+
+        # Mock a row with FSMField data
+        row_data = {"state": "ready"}
+
+        self.resource._meta.model = Domain
+
+        instance = self.resource.init_instance(row=row_data)
+
+        # Assert that the instance is initialized correctly
+        self.assertIsInstance(instance, Domain)
+        self.assertEqual(instance.state, "ready")
+
+    def test_import_field(self):
+        """Test that importing a field does not import FSM field"""
+
+        # Mock a FSMField and a non-FSM-field
+        fsm_field_mock = Mock(attribute="state", column_name="state")
+        field_mock = Mock(attribute="name", column_name="name")
+        # Mock the data
+        data_mock = {"state": "unknown", "name": "test"}
+        # Define a mock Domain
+        obj = Domain(state=Domain.State.UNKNOWN, name="test")
+
+        # Mock the save() method of fields so that we can test if save is called
+        # save() is only supposed to be called for non FSM fields
+        field_mock.save = Mock()
+        fsm_field_mock.save = Mock()
+
+        # Call the method with FSMField and non-FSMField
+        self.resource.import_field(fsm_field_mock, obj, data=data_mock, is_m2m=False)
+        self.resource.import_field(field_mock, obj, data=data_mock, is_m2m=False)
+
+        # Assert that field.save() in super().import_field() is called only for non-FSMField
+        field_mock.save.assert_called_once()
+        fsm_field_mock.save.assert_not_called()
+
+
 class TestDomainAdmin(MockEppLib, WebTest):
     # csrf checks do not work with WebTest.
     # We disable them here. TODO for another ticket.
@@ -600,10 +644,9 @@ class TestDomainAdmin(MockEppLib, WebTest):
                 domain_request.approve()
 
             response = self.client.get("/admin/registrar/domain/")
-
             # There are 4 template references to Federal (4) plus four references in the table
             # for our actual domain_request
-            self.assertContains(response, "Federal", count=42)
+            self.assertContains(response, "Federal", count=54)
             # This may be a bit more robust
             self.assertContains(response, '<td class="field-generic_org_type">Federal</td>', count=1)
             # Now let's make sure the long description does not exist
@@ -852,6 +895,14 @@ class TestDomainAdmin(MockEppLib, WebTest):
     def test_place_and_remove_hold_epp(self):
         raise
 
+    @override_settings(IS_PRODUCTION=True)
+    def test_prod_only_shows_export(self):
+        """Test that production environment only displays export"""
+        with less_console_noise():
+            response = self.client.get("/admin/registrar/domain/")
+            self.assertContains(response, ">Export<")
+            self.assertNotContains(response, ">Import<")
+
     def tearDown(self):
         super().tearDown()
         PublicContact.objects.all().delete()
@@ -1376,7 +1427,7 @@ class TestDomainRequestAdmin(MockEppLib):
             response = self.client.get("/admin/registrar/domainrequest/?generic_org_type__exact=federal")
             # There are 2 template references to Federal (4) and two in the results data
             # of the request
-            self.assertContains(response, "Federal", count=40)
+            self.assertContains(response, "Federal", count=52)
             # This may be a bit more robust
             self.assertContains(response, '<td class="field-generic_org_type">Federal</td>', count=1)
             # Now let's make sure the long description does not exist

src/requirements.txt

@@ -10,6 +10,7 @@ cffi==1.16.0; platform_python_implementation != 'PyPy'
 charset-normalizer==3.3.2; python_full_version >= '3.7.0'
 cryptography==42.0.5; python_version >= '3.7'
 defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+diff-match-patch==20230430; python_version >= '3.7'
 dj-database-url==2.1.0
 dj-email-url==1.0.6
 django==4.2.10; python_version >= '3.8'
@@ -20,11 +21,13 @@ django-cache-url==3.4.5
 django-cors-headers==4.3.1; python_version >= '3.8'
 django-csp==3.8
 django-fsm==2.8.1
+django-import-export==3.3.8; python_version >= '3.8'
 django-login-required-middleware==0.9.0
 django-phonenumber-field[phonenumberslite]==7.3.0; python_version >= '3.8'
 django-waffle==4.1.0; python_version >= '3.8'
 django-widget-tweaks==1.5.0; python_version >= '3.8'
 environs[django]==11.0.0; python_version >= '3.8'
+et-xmlfile==1.1.0; python_version >= '3.6'
 faker==25.0.0; python_version >= '3.8'
 fred-epplib@ git+https://github.com/cisagov/epplib.git@d56d183f1664f34c40ca9716a3a9a345f0ef561c
 furl==2.1.3
@@ -36,9 +39,12 @@ idna==3.7; python_version >= '3.5'
 jmespath==1.0.1; python_version >= '3.7'
 lxml==5.2.1; python_version >= '3.6'
 mako==1.3.3; python_version >= '3.8'
+markuppy==1.14
 markupsafe==2.1.5; python_version >= '3.7'
 marshmallow==3.21.1; python_version >= '3.8'
+odfpy==1.4.1
 oic==1.7.0; python_version ~= '3.8'
+openpyxl==3.1.2
 orderedmultidict==1.0.1
 packaging==24.0; python_version >= '3.7'
 phonenumberslite==8.13.35
@@ -51,15 +57,19 @@ pydantic-settings==2.2.1; python_version >= '3.8'
 pyjwkest==1.4.2
 python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 python-dotenv==1.0.1; python_version >= '3.8'
+pyyaml==6.0.1
 pyzipper==0.3.6; python_version >= '3.4'
 requests==2.31.0; python_version >= '3.7'
 s3transfer==0.10.1; python_version >= '3.8'
 setuptools==69.5.1; python_version >= '3.8'
 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sqlparse==0.5.0; python_version >= '3.8'
+tablib[html,ods,xls,xlsx,yaml]==3.5.0; python_version >= '3.8'
 tblib==3.0.0; python_version >= '3.8'
 typing-extensions==4.11.0; python_version >= '3.8'
 urllib3==2.2.1; python_version >= '3.8'
 whitenoise==6.6.0; python_version >= '3.8'
+xlrd==2.0.1
+xlwt==1.3.0
 zope.event==5.0; python_version >= '3.7'
 zope.interface==6.3; python_version >= '3.7'