From 624871de2da88ccac71e080b1b88a316604d349f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 14 Nov 2023 15:00:03 -0500 Subject: [PATCH 001/119] error handling for generic error messages in nameserver, dsdata and security email forms --- src/registrar/utility/errors.py | 67 +++++++++++++++++++++++++++++++++ src/registrar/views/domain.py | 12 ++++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 4ca3a9a12..67e6d9de2 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -69,6 +69,7 @@ class NameserverErrorCodes(IntEnum): - 5 UNABLE_TO_UPDATE_DOMAIN unable to update the domain - 6 MISSING_HOST host is missing for a nameserver - 7 INVALID_HOST host is invalid for a nameserver + - 8 BAD_DATA bad data input for nameserver """ MISSING_IP = 1 @@ -78,6 +79,7 @@ class NameserverErrorCodes(IntEnum): UNABLE_TO_UPDATE_DOMAIN = 5 MISSING_HOST = 6 INVALID_HOST = 7 + BAD_DATA = 8 class NameserverError(Exception): @@ -96,6 +98,9 @@ class NameserverError(Exception): ), NameserverErrorCodes.MISSING_HOST: ("Name server must be provided to enter IP address."), NameserverErrorCodes.INVALID_HOST: ("Enter a name server in the required format, like ns1.example.com"), + NameserverErrorCodes.BAD_DATA: ( + "There’s something wrong with the name server information you provided. If you need help email us at help@get.gov." + ), } def __init__(self, *args, code=None, nameserver=None, ip=None, **kwargs): @@ -112,3 +117,65 @@ class NameserverError(Exception): def __str__(self): return f"{self.message}" + + +class DsDataErrorCodes(IntEnum): + """Used in the DsDataError class for + error mapping. + Overview of ds data error codes: + - 1 BAD_DATA bad data input in ds data + """ + + BAD_DATA = 1 + + +class DsDataError(Exception): + """ + DsDataError class used to raise exceptions on + the ds data getter + """ + + _error_mapping = { + DsDataErrorCodes.BAD_DATA: ( + "There’s something wrong with the DS data you provided. If you need help email us at help@get.gov." + ) + } + + def __init__(self, *args, code=None, **kwargs): + super().__init__(*args, **kwargs) + self.code = code + if self.code in self._error_mapping: + self.message = self._error_mapping.get(self.code) + + def __str__(self): + return f"{self.message}" + + +class SecurityEmailErrorCodes(IntEnum): + """Used in the SecurityEmailError class for + error mapping. + Overview of security email error codes: + - 1 BAD_DATA bad data input in security email + """ + + BAD_DATA = 1 + + +class SecurityEmailError(Exception): + """ + SecurityEmailError class used to raise exceptions on + the security email form + """ + + _error_mapping = { + SecurityEmailErrorCodes.BAD_DATA: ("Enter an email address in the required format, like name@example.com.") + } + + def __init__(self, *args, code=None, **kwargs): + super().__init__(*args, **kwargs) + self.code = code + if self.code in self._error_mapping: + self.message = self._error_mapping.get(self.code) + + def __str__(self): + return f"{self.message}" diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 88fad1567..f365c284a 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -28,6 +28,10 @@ from registrar.utility.errors import ( GenericErrorCodes, NameserverError, NameserverErrorCodes as nsErrorCodes, + DsDataError, + DsDataErrorCodes, + SecurityEmailError, + SecurityEmailErrorCodes, ) from registrar.models.utility.contact_error import ContactError @@ -315,7 +319,7 @@ class DomainNameserversView(DomainFormBaseView): ) logger.error(f"Registry connection error: {Err}") else: - messages.error(self.request, GenericError(code=GenericErrorCodes.GENERIC_ERROR)) + messages.error(self.request, NameserverError(code=nsErrorCodes.BAD_DATA)) logger.error(f"Registry error: {Err}") else: messages.success( @@ -491,7 +495,7 @@ class DomainDsDataView(DomainFormBaseView): ) logger.error(f"Registry connection error: {err}") else: - messages.error(self.request, GenericError(code=GenericErrorCodes.GENERIC_ERROR)) + messages.error(self.request, DsDataError(code=DsDataErrorCodes.BAD_DATA)) logger.error(f"Registry error: {err}") return self.form_invalid(formset) else: @@ -581,10 +585,10 @@ class DomainSecurityEmailView(DomainFormBaseView): ) logger.error(f"Registry connection error: {Err}") else: - messages.error(self.request, GenericError(code=GenericErrorCodes.GENERIC_ERROR)) + messages.error(self.request, SecurityEmailError(code=SecurityEmailErrorCodes.BAD_DATA)) logger.error(f"Registry error: {Err}") except ContactError as Err: - messages.error(self.request, GenericError(code=GenericErrorCodes.GENERIC_ERROR)) + messages.error(self.request, SecurityEmailError(code=SecurityEmailErrorCodes.BAD_DATA)) logger.error(f"Generic registry error: {Err}") else: messages.success(self.request, "The security email for this domain has been updated.") From 99083fff3ffb11593dffdadb5ce0aad715af71b5 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 14 Nov 2023 16:35:27 -0500 Subject: [PATCH 002/119] formatted for linter --- src/registrar/utility/errors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 67e6d9de2..13506e8a0 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -99,7 +99,8 @@ class NameserverError(Exception): NameserverErrorCodes.MISSING_HOST: ("Name server must be provided to enter IP address."), NameserverErrorCodes.INVALID_HOST: ("Enter a name server in the required format, like ns1.example.com"), NameserverErrorCodes.BAD_DATA: ( - "There’s something wrong with the name server information you provided. If you need help email us at help@get.gov." + "There’s something wrong with the name server information you provided. " + "If you need help email us at help@get.gov." ), } From 00916fa030ffa52f06eac52edc1df0d5a782868f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 14 Nov 2023 16:39:21 -0500 Subject: [PATCH 003/119] fixed test case --- src/registrar/tests/test_views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 178b40e51..f8d17b832 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1740,7 +1740,11 @@ and try again. If you continue to receive this error after a few tries, contact help@get.gov """, ), - ("ContactError", form_data_contact_error, "Value entered was wrong."), + ( + "ContactError", + form_data_contact_error, + "Enter an email address in the required format, like name@example.com." + ), ( "RegistrySuccess", form_data_success, From 3c421475b9a422154e38afed176e65447bc949d1 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 14 Nov 2023 16:53:57 -0500 Subject: [PATCH 004/119] formatted for linter --- src/registrar/tests/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index f8d17b832..ae7655187 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1743,7 +1743,7 @@ contact help@get.gov ( "ContactError", form_data_contact_error, - "Enter an email address in the required format, like name@example.com." + "Enter an email address in the required format, like name@example.com.", ), ( "RegistrySuccess", From 0b826eb93426b165e8a42be5d49f678ed5ecc1e7 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Nov 2023 05:28:20 -0500 Subject: [PATCH 005/119] modified tests to use error file instead of hardcoded error message in multiple files --- src/registrar/tests/test_views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index ae7655187..8f8e161a2 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -13,6 +13,10 @@ import boto3_mocking # type: ignore from registrar.utility.errors import ( NameserverError, NameserverErrorCodes, + SecurityEmailError, + SecurityEmailErrorCodes, + GenericError, + GenericErrorCodes, ) from registrar.models import ( @@ -1734,16 +1738,12 @@ class TestDomainSecurityEmail(TestDomainOverview): ( "RegistryError", form_data_registry_error, - """ -We’re experiencing a system connection error. Please wait a few minutes -and try again. If you continue to receive this error after a few tries, -contact help@get.gov - """, + str(GenericError(code=GenericErrorCodes.CANNOT_CONTACT_REGISTRY)), ), ( "ContactError", form_data_contact_error, - "Enter an email address in the required format, like name@example.com.", + str(SecurityEmailError(code=SecurityEmailErrorCodes.BAD_DATA)), ), ( "RegistrySuccess", From 4cdf605bf2c14344f4e3edbf6d87fd41c4c486bd Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Nov 2023 06:09:35 -0500 Subject: [PATCH 006/119] wip --- src/registrar/forms/domain.py | 86 +++++++++++++++++++++++++++++++++-- src/registrar/views/domain.py | 2 +- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 6cb5b338f..54df80b14 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -16,6 +16,8 @@ from .common import ( DIGEST_TYPE_CHOICES, ) +import re + class DomainAddUserForm(forms.Form): """Form for adding a user to a domain.""" @@ -228,12 +230,16 @@ class DomainDnssecForm(forms.Form): class DomainDsdataForm(forms.Form): """Form for adding or editing DNSSEC DS Data to a domain.""" + def validate_hexadecimal(value): + if not re.match(r'^[0-9a-fA-F]+$', value): + raise forms.ValidationError('Digest must contain only alphanumeric characters [0-9,a-f].') + key_tag = forms.IntegerField( required=True, label="Key tag", validators=[ - MinValueValidator(0, message="Value must be between 0 and 65535"), - MaxValueValidator(65535, message="Value must be between 0 and 65535"), + MinValueValidator(0, message="Key tag must be less than 65535"), + MaxValueValidator(65535, message="Key tag must be less than 65535"), ], error_messages={"required": ("Key tag is required.")}, ) @@ -257,9 +263,83 @@ class DomainDsdataForm(forms.Form): digest = forms.CharField( required=True, label="Digest", - error_messages={"required": ("Digest is required.")}, + validators=[validate_hexadecimal], + max_length=64, + error_messages={ + "required": "Digest is required.", + "max_length": "Digest must be at most 64 characters long.", + }, ) + def clean(self): + # clean is called from clean_forms, which is called from is_valid + # after clean_fields. it is used to determine form level errors. + # is_valid is typically called from view during a post + cleaned_data = super().clean() + digest_type = cleaned_data.get("digest_type", 0) + digest = cleaned_data.get("digest", "") + # validate length of digest depending on digest_type + if digest_type == 1 and len(digest) != 40: + self.add_error( + "digest", + DsDa) + # remove ANY spaces in the server field + server = server.replace(" ", "") + # lowercase the server + server = server.lower() + cleaned_data["server"] = server + ip = cleaned_data.get("ip", None) + # remove ANY spaces in the ip field + ip = ip.replace(" ", "") + domain = cleaned_data.get("domain", "") + + ip_list = self.extract_ip_list(ip) + + # validate if the form has a server or an ip + if (ip and ip_list) or server: + self.validate_nameserver_ip_combo(domain, server, ip_list) + + return cleaned_data + + def clean_empty_strings(self, cleaned_data): + ip = cleaned_data.get("ip", "") + if ip and len(ip.strip()) == 0: + cleaned_data["ip"] = None + + def extract_ip_list(self, ip): + return [ip.strip() for ip in ip.split(",")] if ip else [] + + def validate_nameserver_ip_combo(self, domain, server, ip_list): + try: + Domain.checkHostIPCombo(domain, server, ip_list) + except NameserverError as e: + if e.code == nsErrorCodes.GLUE_RECORD_NOT_ALLOWED: + self.add_error( + "server", + NameserverError( + code=nsErrorCodes.GLUE_RECORD_NOT_ALLOWED, + nameserver=domain, + ip=ip_list, + ), + ) + elif e.code == nsErrorCodes.MISSING_IP: + self.add_error( + "ip", + NameserverError(code=nsErrorCodes.MISSING_IP, nameserver=domain, ip=ip_list), + ) + elif e.code == nsErrorCodes.MISSING_HOST: + self.add_error( + "server", + NameserverError(code=nsErrorCodes.MISSING_HOST, nameserver=domain, ip=ip_list), + ) + elif e.code == nsErrorCodes.INVALID_HOST: + self.add_error( + "server", + NameserverError(code=nsErrorCodes.INVALID_HOST, nameserver=server, ip=ip_list), + ) + else: + self.add_error("ip", str(e)) + DomainDsdataFormset = formset_factory( DomainDsdataForm, diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 88fad1567..7b8997de0 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -445,7 +445,7 @@ class DomainDsDataView(DomainFormBaseView): modal_button = ( '' + 'name="disable-override-click">Remove all DS Data' ) # context to back out of a broken form on all fields delete From 48caa775dddaeb82c67945e545aa954ab04ee6e6 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Nov 2023 06:54:35 -0500 Subject: [PATCH 007/119] digest validation; error messages; test cases --- src/registrar/forms/domain.py | 75 ++++--------------- src/registrar/tests/test_views.py | 118 +++++++++++++++++++++++++++++- src/registrar/utility/errors.py | 27 ++++++- 3 files changed, 155 insertions(+), 65 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 54df80b14..49a1f0f3c 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -8,6 +8,8 @@ from phonenumber_field.widgets import RegionalPhoneNumberWidget from registrar.utility.errors import ( NameserverError, NameserverErrorCodes as nsErrorCodes, + DsDataError, + DsDataErrorCodes, ) from ..models import Contact, DomainInformation, Domain @@ -232,14 +234,14 @@ class DomainDsdataForm(forms.Form): def validate_hexadecimal(value): if not re.match(r'^[0-9a-fA-F]+$', value): - raise forms.ValidationError('Digest must contain only alphanumeric characters [0-9,a-f].') + raise forms.ValidationError(str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_CHARS))) key_tag = forms.IntegerField( required=True, label="Key tag", validators=[ - MinValueValidator(0, message="Key tag must be less than 65535"), - MaxValueValidator(65535, message="Key tag must be less than 65535"), + MinValueValidator(0, message=str(DsDataError(code=DsDataErrorCodes.INVALID_KEYTAG_SIZE))), + MaxValueValidator(65535, message=str(DsDataError(code=DsDataErrorCodes.INVALID_KEYTAG_SIZE))), ], error_messages={"required": ("Key tag is required.")}, ) @@ -257,7 +259,7 @@ class DomainDsdataForm(forms.Form): label="Digest type", coerce=int, # need to coerce into int so dsData objects can be compared choices=[(None, "--Select--")] + DIGEST_TYPE_CHOICES, # type: ignore - error_messages={"required": ("Digest Type is required.")}, + error_messages={"required": ("Digest type is required.")}, ) digest = forms.CharField( @@ -267,7 +269,7 @@ class DomainDsdataForm(forms.Form): max_length=64, error_messages={ "required": "Digest is required.", - "max_length": "Digest must be at most 64 characters long.", + "max_length": str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_LENGTH)), }, ) @@ -282,64 +284,15 @@ class DomainDsdataForm(forms.Form): if digest_type == 1 and len(digest) != 40: self.add_error( "digest", - DsDa) - # remove ANY spaces in the server field - server = server.replace(" ", "") - # lowercase the server - server = server.lower() - cleaned_data["server"] = server - ip = cleaned_data.get("ip", None) - # remove ANY spaces in the ip field - ip = ip.replace(" ", "") - domain = cleaned_data.get("domain", "") - - ip_list = self.extract_ip_list(ip) - - # validate if the form has a server or an ip - if (ip and ip_list) or server: - self.validate_nameserver_ip_combo(domain, server, ip_list) - + DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA1), + ) + elif digest_type == 2 and len(digest) != 64: + self.add_error( + "digest", + DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA256), + ) return cleaned_data - def clean_empty_strings(self, cleaned_data): - ip = cleaned_data.get("ip", "") - if ip and len(ip.strip()) == 0: - cleaned_data["ip"] = None - - def extract_ip_list(self, ip): - return [ip.strip() for ip in ip.split(",")] if ip else [] - - def validate_nameserver_ip_combo(self, domain, server, ip_list): - try: - Domain.checkHostIPCombo(domain, server, ip_list) - except NameserverError as e: - if e.code == nsErrorCodes.GLUE_RECORD_NOT_ALLOWED: - self.add_error( - "server", - NameserverError( - code=nsErrorCodes.GLUE_RECORD_NOT_ALLOWED, - nameserver=domain, - ip=ip_list, - ), - ) - elif e.code == nsErrorCodes.MISSING_IP: - self.add_error( - "ip", - NameserverError(code=nsErrorCodes.MISSING_IP, nameserver=domain, ip=ip_list), - ) - elif e.code == nsErrorCodes.MISSING_HOST: - self.add_error( - "server", - NameserverError(code=nsErrorCodes.MISSING_HOST, nameserver=domain, ip=ip_list), - ) - elif e.code == nsErrorCodes.INVALID_HOST: - self.add_error( - "server", - NameserverError(code=nsErrorCodes.INVALID_HOST, nameserver=server, ip=ip_list), - ) - else: - self.add_error("ip", str(e)) - DomainDsdataFormset = formset_factory( DomainDsdataForm, diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 8f8e161a2..763f9a2ac 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -17,6 +17,8 @@ from registrar.utility.errors import ( SecurityEmailErrorCodes, GenericError, GenericErrorCodes, + DsDataError, + DsDataErrorCodes, ) from registrar.models import ( @@ -1878,7 +1880,30 @@ class TestDomainDNSSEC(TestDomainOverview): self.assertContains(page, "The DS Data records for this domain have been updated.") def test_ds_data_form_invalid(self): - """DS Data form errors with invalid data + """DS Data form errors with invalid data (missing required fields) + + Uses self.app WebTest because we need to interact with forms. + """ + add_data_page = self.app.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_dsdata.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # all four form fields are required, so will test with each blank + add_data_page.forms[0]["form-0-key_tag"] = "" + add_data_page.forms[0]["form-0-algorithm"] = "" + add_data_page.forms[0]["form-0-digest_type"] = "" + add_data_page.forms[0]["form-0-digest"] = "" + with less_console_noise(): # swallow logged warning message + result = add_data_page.forms[0].submit() + # form submission was a post with an error, response should be a 200 + # error text appears twice, once at the top of the page, once around + # the field. + self.assertContains(result, "Key tag is required", count=2, status_code=200) + self.assertContains(result, "Algorithm is required", count=2, status_code=200) + self.assertContains(result, "Digest type is required", count=2, status_code=200) + self.assertContains(result, "Digest is required", count=2, status_code=200) + + def test_ds_data_form_invalid_keytag(self): + """DS Data form errors with invalid data (key tag too large) Uses self.app WebTest because we need to interact with forms. """ @@ -1887,13 +1912,100 @@ class TestDomainDNSSEC(TestDomainOverview): self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) # first two nameservers are required, so if we empty one out we should # get a form error - add_data_page.forms[0]["form-0-key_tag"] = "" + add_data_page.forms[0]["form-0-key_tag"] = "65536" # > 65535 + add_data_page.forms[0]["form-0-algorithm"] = "" + add_data_page.forms[0]["form-0-digest_type"] = "" + add_data_page.forms[0]["form-0-digest"] = "" with less_console_noise(): # swallow logged warning message result = add_data_page.forms[0].submit() # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the field. - self.assertContains(result, "Key tag is required", count=2, status_code=200) + self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_KEYTAG_SIZE)), count=2, status_code=200) + + def test_ds_data_form_invalid_digest_length(self): + """DS Data form errors with invalid data (digest too long) + + Uses self.app WebTest because we need to interact with forms. + """ + add_data_page = self.app.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_dsdata.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # first two nameservers are required, so if we empty one out we should + # get a form error + add_data_page.forms[0]["form-0-key_tag"] = "1234" + add_data_page.forms[0]["form-0-algorithm"] = "3" + add_data_page.forms[0]["form-0-digest_type"] = "1" + add_data_page.forms[0]["form-0-digest"] = "1234567890123456789012345678901234567890123456789012345678901234567890" + with less_console_noise(): # swallow logged warning message + result = add_data_page.forms[0].submit() + # form submission was a post with an error, response should be a 200 + # error text appears twice, once at the top of the page, once around + # the field. + self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_LENGTH)), count=2, status_code=200) + + def test_ds_data_form_invalid_digest_chars(self): + """DS Data form errors with invalid data (digest contains non hexadecimal chars) + + Uses self.app WebTest because we need to interact with forms. + """ + add_data_page = self.app.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_dsdata.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # first two nameservers are required, so if we empty one out we should + # get a form error + add_data_page.forms[0]["form-0-key_tag"] = "1234" + add_data_page.forms[0]["form-0-algorithm"] = "3" + add_data_page.forms[0]["form-0-digest_type"] = "1" + add_data_page.forms[0]["form-0-digest"] = "GG1234" + with less_console_noise(): # swallow logged warning message + result = add_data_page.forms[0].submit() + # form submission was a post with an error, response should be a 200 + # error text appears twice, once at the top of the page, once around + # the field. + self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_CHARS)), count=2, status_code=200) + + def test_ds_data_form_invalid_digest_sha1(self): + """DS Data form errors with invalid data (digest is invalid sha-1) + + Uses self.app WebTest because we need to interact with forms. + """ + add_data_page = self.app.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_dsdata.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # first two nameservers are required, so if we empty one out we should + # get a form error + add_data_page.forms[0]["form-0-key_tag"] = "1234" + add_data_page.forms[0]["form-0-algorithm"] = "3" + add_data_page.forms[0]["form-0-digest_type"] = "1" # SHA-1 + add_data_page.forms[0]["form-0-digest"] = "A123" + with less_console_noise(): # swallow logged warning message + result = add_data_page.forms[0].submit() + # form submission was a post with an error, response should be a 200 + # error text appears twice, once at the top of the page, once around + # the field. + self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA1)), count=2, status_code=200) + + def test_ds_data_form_invalid_digest_sha256(self): + """DS Data form errors with invalid data (digest is invalid sha-256) + + Uses self.app WebTest because we need to interact with forms. + """ + add_data_page = self.app.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_dsdata.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # first two nameservers are required, so if we empty one out we should + # get a form error + add_data_page.forms[0]["form-0-key_tag"] = "1234" + add_data_page.forms[0]["form-0-algorithm"] = "3" + add_data_page.forms[0]["form-0-digest_type"] = "2" # SHA-256 + add_data_page.forms[0]["form-0-digest"] = "GG1234" + with less_console_noise(): # swallow logged warning message + result = add_data_page.forms[0].submit() + # form submission was a post with an error, response should be a 200 + # error text appears twice, once at the top of the page, once around + # the field. + self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA256)), count=2, status_code=200) class TestApplicationStatus(TestWithUser, WebTest): diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 13506e8a0..50d62933c 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -125,9 +125,19 @@ class DsDataErrorCodes(IntEnum): error mapping. Overview of ds data error codes: - 1 BAD_DATA bad data input in ds data + - 2 INVALID_DIGEST_SHA1 invalid digest for digest type SHA-1 + - 3 INVALID_DIGEST_SHA256 invalid digest for digest type SHA-256 + - 4 INVALID_DIGEST_LENGTH invalid digest length exceeds 64 + - 5 INVALID_DIGEST_CHARS invalid chars in digest + - 6 INVALID_KEYTAG_SIZE invalid key tag size > 65535 """ BAD_DATA = 1 + INVALID_DIGEST_SHA1 = 2 + INVALID_DIGEST_SHA256 = 3 + INVALID_DIGEST_LENGTH = 4 + INVALID_DIGEST_CHARS = 5 + INVALID_KEYTAG_SIZE = 6 class DsDataError(Exception): @@ -139,7 +149,22 @@ class DsDataError(Exception): _error_mapping = { DsDataErrorCodes.BAD_DATA: ( "There’s something wrong with the DS data you provided. If you need help email us at help@get.gov." - ) + ), + DsDataErrorCodes.INVALID_DIGEST_SHA1: ( + "SHA-1 digest must be exactly 40 characters." + ), + DsDataErrorCodes.INVALID_DIGEST_SHA256: ( + "SHA-256 digest must be exactly 64 characters." + ), + DsDataErrorCodes.INVALID_DIGEST_LENGTH: ( + "Digest must be at most 64 characters." + ), + DsDataErrorCodes.INVALID_DIGEST_CHARS: ( + "Digest must contain only alphanumeric characters [0-9,a-f]." + ), + DsDataErrorCodes.INVALID_KEYTAG_SIZE: ( + "Key tag must be less than 65535" + ), } def __init__(self, *args, code=None, **kwargs): From 9785bed0655b5c6faa6818d4190f3ac650873866 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 15 Nov 2023 07:50:17 -0500 Subject: [PATCH 008/119] formatted for linter --- src/registrar/forms/domain.py | 2 +- src/registrar/tests/test_views.py | 24 ++++++++++++++++++------ src/registrar/utility/errors.py | 20 +++++--------------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 49a1f0f3c..1ef6ce354 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -233,7 +233,7 @@ class DomainDsdataForm(forms.Form): """Form for adding or editing DNSSEC DS Data to a domain.""" def validate_hexadecimal(value): - if not re.match(r'^[0-9a-fA-F]+$', value): + if not re.match(r"^[0-9a-fA-F]+$", value): raise forms.ValidationError(str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_CHARS))) key_tag = forms.IntegerField( diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 763f9a2ac..334fd2824 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1921,7 +1921,9 @@ class TestDomainDNSSEC(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the field. - self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_KEYTAG_SIZE)), count=2, status_code=200) + self.assertContains( + result, str(DsDataError(code=DsDataErrorCodes.INVALID_KEYTAG_SIZE)), count=2, status_code=200 + ) def test_ds_data_form_invalid_digest_length(self): """DS Data form errors with invalid data (digest too long) @@ -1936,13 +1938,17 @@ class TestDomainDNSSEC(TestDomainOverview): add_data_page.forms[0]["form-0-key_tag"] = "1234" add_data_page.forms[0]["form-0-algorithm"] = "3" add_data_page.forms[0]["form-0-digest_type"] = "1" - add_data_page.forms[0]["form-0-digest"] = "1234567890123456789012345678901234567890123456789012345678901234567890" + add_data_page.forms[0][ + "form-0-digest" + ] = "1234567890123456789012345678901234567890123456789012345678901234567890" with less_console_noise(): # swallow logged warning message result = add_data_page.forms[0].submit() # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the field. - self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_LENGTH)), count=2, status_code=200) + self.assertContains( + result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_LENGTH)), count=2, status_code=200 + ) def test_ds_data_form_invalid_digest_chars(self): """DS Data form errors with invalid data (digest contains non hexadecimal chars) @@ -1963,7 +1969,9 @@ class TestDomainDNSSEC(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the field. - self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_CHARS)), count=2, status_code=200) + self.assertContains( + result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_CHARS)), count=2, status_code=200 + ) def test_ds_data_form_invalid_digest_sha1(self): """DS Data form errors with invalid data (digest is invalid sha-1) @@ -1984,7 +1992,9 @@ class TestDomainDNSSEC(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the field. - self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA1)), count=2, status_code=200) + self.assertContains( + result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA1)), count=2, status_code=200 + ) def test_ds_data_form_invalid_digest_sha256(self): """DS Data form errors with invalid data (digest is invalid sha-256) @@ -2005,7 +2015,9 @@ class TestDomainDNSSEC(TestDomainOverview): # form submission was a post with an error, response should be a 200 # error text appears twice, once at the top of the page, once around # the field. - self.assertContains(result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA256)), count=2, status_code=200) + self.assertContains( + result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_SHA256)), count=2, status_code=200 + ) class TestApplicationStatus(TestWithUser, WebTest): diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 50d62933c..2f9daa37d 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -150,21 +150,11 @@ class DsDataError(Exception): DsDataErrorCodes.BAD_DATA: ( "There’s something wrong with the DS data you provided. If you need help email us at help@get.gov." ), - DsDataErrorCodes.INVALID_DIGEST_SHA1: ( - "SHA-1 digest must be exactly 40 characters." - ), - DsDataErrorCodes.INVALID_DIGEST_SHA256: ( - "SHA-256 digest must be exactly 64 characters." - ), - DsDataErrorCodes.INVALID_DIGEST_LENGTH: ( - "Digest must be at most 64 characters." - ), - DsDataErrorCodes.INVALID_DIGEST_CHARS: ( - "Digest must contain only alphanumeric characters [0-9,a-f]." - ), - DsDataErrorCodes.INVALID_KEYTAG_SIZE: ( - "Key tag must be less than 65535" - ), + DsDataErrorCodes.INVALID_DIGEST_SHA1: ("SHA-1 digest must be exactly 40 characters."), + DsDataErrorCodes.INVALID_DIGEST_SHA256: ("SHA-256 digest must be exactly 64 characters."), + DsDataErrorCodes.INVALID_DIGEST_LENGTH: ("Digest must be at most 64 characters."), + DsDataErrorCodes.INVALID_DIGEST_CHARS: ("Digest must contain only alphanumeric characters [0-9,a-f]."), + DsDataErrorCodes.INVALID_KEYTAG_SIZE: ("Key tag must be less than 65535"), } def __init__(self, *args, code=None, **kwargs): From dd3ea3870c0b683414c4fbf2f90b98b65dd9ebb9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Nov 2023 10:13:41 -0700 Subject: [PATCH 009/119] Update README.md --- docs/developer/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/developer/README.md b/docs/developer/README.md index c23671aac..bee00f4b1 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -293,3 +293,17 @@ it may help to resync your laptop with time.nist.gov: ``` sudo sntp -sS time.nist.gov ``` + +## Test if our connection pool is running +Our connection pool has a built-in `pool_status` object which you can call at anytime to assess the current connection status of the pool. Follow these steps to access it. + +1. `cf ssh getgov-{env-name} -i {instance-index}` +* env-name -> Which environment to target, e.g. `staging` +* instance-index -> Which instance to target. For instance, `cf ssh getgov-staging -i 0` +2. `/tmp/lifecycle/shell` +3. `./manage.py shell` +4. `from epplibwrapper import CLIENT as registry, commands` +5. `print(registry.pool_status.connection_success)` +* (Should return true) + +If you have multiple instances (staging for example), then repeat commands 1-5 for each instance you want to test. \ No newline at end of file From 240507246a71e0c96e08155a4d58e49354778ac9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 15 Nov 2023 13:46:46 -0700 Subject: [PATCH 010/119] Update README.md --- docs/developer/README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/developer/README.md b/docs/developer/README.md index bee00f4b1..268bbe1bd 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -294,7 +294,20 @@ it may help to resync your laptop with time.nist.gov: sudo sntp -sS time.nist.gov ``` -## Test if our connection pool is running +## Connection pool +To handle our connection to the registry, we utilize a connection pool to keep a socket open to increase responsiveness. In order to accomplish this, We are utilizing a heavily modified version of the (geventconnpool)[https://github.com/rasky/geventconnpool] library. + +### Settings +The config for the connection pool exists inside the `settings.py` file. +| Name | Purpose | +| -------- | ------- | +| EPP_CONNECTION_POOL_SIZE | Determines the number of concurrent sockets that should exist in the pool. | +| POOL_KEEP_ALIVE | Determines the interval in which we ping open connections in seconds. Calculated as POOL_KEEP_ALIVE / EPP_CONNECTION_POOL_SIZE | +| POOL_TIMEOUT | Determines how long we try to keep a pool alive for, before restarting it. | + +Consider updating the `POOL_TIMEOUT` or `POOL_KEEP_ALIVE` periods if the pool often restarts. If the pool only restarts after a period of inactivity, update `POOL_KEEP_ALIVE`. If it restarts during the EPP call itself, then `POOL_TIMEOUT` needs to be updated. + +### Test if the connection pool is running Our connection pool has a built-in `pool_status` object which you can call at anytime to assess the current connection status of the pool. Follow these steps to access it. 1. `cf ssh getgov-{env-name} -i {instance-index}` @@ -304,6 +317,6 @@ Our connection pool has a built-in `pool_status` object which you can call at an 3. `./manage.py shell` 4. `from epplibwrapper import CLIENT as registry, commands` 5. `print(registry.pool_status.connection_success)` -* (Should return true) +* Should return true If you have multiple instances (staging for example), then repeat commands 1-5 for each instance you want to test. \ No newline at end of file From 944c82b9f9c028803b797333615487126b77d402 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Nov 2023 08:39:05 -0500 Subject: [PATCH 011/119] made expiration date readonly in django admin; expiration date setter not implemented; implemented renew_domain; wrote unit tests --- src/registrar/admin.py | 2 +- src/registrar/models/domain.py | 41 ++++++++++++++++++++++- src/registrar/tests/common.py | 18 ++++++++++ src/registrar/tests/test_models_domain.py | 37 +++++++++++++++++++- 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 9de5f563c..c059e5674 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -736,7 +736,7 @@ class DomainAdmin(ListHeaderAdmin): search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" change_list_template = "django/admin/domain_change_list.html" - readonly_fields = ["state"] + readonly_fields = ["state", "expiration_date"] def export_data_type(self, request): # match the CSV example with all the fields diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index cb3b784e9..005682e23 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -216,7 +216,46 @@ class Domain(TimeStampedModel, DomainHelper): @registry_expiration_date.setter # type: ignore def registry_expiration_date(self, ex_date: date): - pass + """ + Direct setting of the expiration date in the registry is not implemented. + + To update the expiration date, use renew_domain method.""" + raise NotImplementedError() + + def renew_domain(self, length: int = 1, unit: epp.Unit = epp.Unit.YEAR): + """ + Renew the domain to a length and unit of time relative to the current + expiration date. + + Default length and unit of time are 1 year. + """ + # if no expiration date from registry, set to today + try: + cur_exp_date = self.registry_expiration_date + except KeyError: + cur_exp_date = date.today() + + # create RenewDomain request + request = commands.RenewDomain( + name=self.name, + cur_exp_date=cur_exp_date, + period = epp.Period(length, unit) + ) + + try: + # update expiration date in registry, and set the updated + # expiration date in the registrar, and in the cache + self._cache["ex_date"] = registry.send(request, cleaned=True).res_data[0].ex_date + self.expiration_date = self._cache["ex_date"] + self.save() + except RegistryError as err: + # if registry error occurs, log the error, and raise it as well + logger.error(f"registry error renewing domain: {err}") + raise (err) + except Exception as e: + # exception raised during the save to registrar + logger.error(f"error updating expiration date in registrar: {e}") + raise (e) @Cache def password(self) -> str: diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 2ad83dfff..3036bb97c 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -556,6 +556,7 @@ class MockEppLib(TestCase): avail=..., addrs=..., registrant=..., + ex_date=..., ): self.auth_info = auth_info self.cr_date = cr_date @@ -565,6 +566,7 @@ class MockEppLib(TestCase): self.avail = avail # use for CheckDomain self.addrs = addrs self.registrant = registrant + self.ex_date = ex_date def dummyInfoContactResultData( self, @@ -811,6 +813,11 @@ class MockEppLib(TestCase): ], ) + mockRenewedDomainExpDate = fakedEppObject( + "fake.gov", + ex_date=datetime.date(2023, 5, 25), + ) + def _mockDomainName(self, _name, _avail=False): return MagicMock( res_data=[ @@ -870,6 +877,8 @@ class MockEppLib(TestCase): return self.mockCheckDomainCommand(_request, cleaned) case commands.DeleteDomain: return self.mockDeleteDomainCommands(_request, cleaned) + case commands.RenewDomain: + return self.mockRenewDomainCommand(_request, cleaned) case _: return MagicMock(res_data=[self.mockDataInfoHosts]) @@ -890,6 +899,15 @@ class MockEppLib(TestCase): raise RegistryError(code=ErrorCode.OBJECT_ASSOCIATION_PROHIBITS_OPERATION) return None + def mockRenewDomainCommand(self, _request, cleaned): + if getattr(_request, "name", None) == "fake-error.gov": + raise RegistryError(code=ErrorCode.PARAMETER_VALUE_RANGE_ERROR) + else: + return MagicMock( + res_data=[self.mockRenewedDomainExpDate], + code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, + ) + def mockInfoDomainCommands(self, _request, cleaned): request_name = getattr(_request, "name", None) diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index 2f2f6d962..8f126b74a 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -56,7 +56,7 @@ class TestDomainCache(MockEppLib): self.assertFalse("avail" in domain._cache.keys()) # using a setter should clear the cache - domain.registry_expiration_date = datetime.date.today() + domain.dnssecdata = [] self.assertEquals(domain._cache, {}) # send should have been called only once @@ -1953,6 +1953,41 @@ class TestRegistrantDNSSEC(MockEppLib): self.assertTrue(err.is_client_error() or err.is_session_error() or err.is_server_error()) +class TestExpirationDate(MockEppLib): + """User may renew expiration date by a number of units of time""" + + def setUp(self): + """ + Domain exists in registry + """ + super().setUp() + # for the tests, need a domain in the ready state + self.domain, _ = Domain.objects.get_or_create(name="fake.gov", state=Domain.State.READY) + # for the test, need a domain that will raise an exception + self.domain_w_error, _ = Domain.objects.get_or_create(name="fake-error.gov", state=Domain.State.READY) + + def tearDown(self): + Domain.objects.all().delete() + super().tearDown() + + def test_expiration_date_setter_not_implemented(self): + """assert that the setter for expiration date is not implemented and will raise error""" + with self.assertRaises(NotImplementedError) as err: + self.domain.registry_expiration_date = datetime.date.today() + + def test_renew_domain(self): + """assert that the renew_domain sets new expiration date in cache and saves to registrar""" + self.domain.renew_domain() + test_date = datetime.date(2023, 5, 25) + self.assertEquals(self.domain._cache["ex_date"], test_date) + self.assertEquals(self.domain.expiration_date, test_date) + + def test_renew_domain_error(self): + """assert that the renew_domain raises an exception when registry raises error""" + with self.assertRaises(RegistryError) as err: + self.domain_w_error.renew_domain() + + class TestAnalystClientHold(MockEppLib): """Rule: Analysts may suspend or restore a domain by using client hold""" From 3107a9213dc638f8c9f7c4492c47901786a8c138 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Nov 2023 08:42:53 -0500 Subject: [PATCH 012/119] formatted code --- src/registrar/models/domain.py | 10 +++------- src/registrar/tests/common.py | 2 +- src/registrar/tests/test_models_domain.py | 6 +++--- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 005682e23..483fc74f9 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -218,7 +218,7 @@ class Domain(TimeStampedModel, DomainHelper): def registry_expiration_date(self, ex_date: date): """ Direct setting of the expiration date in the registry is not implemented. - + To update the expiration date, use renew_domain method.""" raise NotImplementedError() @@ -226,7 +226,7 @@ class Domain(TimeStampedModel, DomainHelper): """ Renew the domain to a length and unit of time relative to the current expiration date. - + Default length and unit of time are 1 year. """ # if no expiration date from registry, set to today @@ -236,11 +236,7 @@ class Domain(TimeStampedModel, DomainHelper): cur_exp_date = date.today() # create RenewDomain request - request = commands.RenewDomain( - name=self.name, - cur_exp_date=cur_exp_date, - period = epp.Period(length, unit) - ) + request = commands.RenewDomain(name=self.name, cur_exp_date=cur_exp_date, period=epp.Period(length, unit)) try: # update expiration date in registry, and set the updated diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 3036bb97c..9a062106f 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -907,7 +907,7 @@ class MockEppLib(TestCase): res_data=[self.mockRenewedDomainExpDate], code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, ) - + def mockInfoDomainCommands(self, _request, cleaned): request_name = getattr(_request, "name", None) diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index 8f126b74a..c75b1b935 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -1972,7 +1972,7 @@ class TestExpirationDate(MockEppLib): def test_expiration_date_setter_not_implemented(self): """assert that the setter for expiration date is not implemented and will raise error""" - with self.assertRaises(NotImplementedError) as err: + with self.assertRaises(NotImplementedError): self.domain.registry_expiration_date = datetime.date.today() def test_renew_domain(self): @@ -1984,10 +1984,10 @@ class TestExpirationDate(MockEppLib): def test_renew_domain_error(self): """assert that the renew_domain raises an exception when registry raises error""" - with self.assertRaises(RegistryError) as err: + with self.assertRaises(RegistryError): self.domain_w_error.renew_domain() - + class TestAnalystClientHold(MockEppLib): """Rule: Analysts may suspend or restore a domain by using client hold""" From 7ec6072a9235b6ad22400d76bf06a26294fb013b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Thu, 16 Nov 2023 10:36:46 -0500 Subject: [PATCH 013/119] adding a warning when expiration date not set --- src/registrar/models/domain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 483fc74f9..94924a698 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -233,6 +233,7 @@ class Domain(TimeStampedModel, DomainHelper): try: cur_exp_date = self.registry_expiration_date except KeyError: + logger.warning("current expiration date not set; setting to today") cur_exp_date = date.today() # create RenewDomain request From 35a0ed751e5b52b3e1c929a21515f9abbb77f371 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:16:01 -0700 Subject: [PATCH 014/119] Parsing org data --- .../commands/load_organization_data.py | 47 +++ .../utility/extra_transition_domain_helper.py | 338 +++++++++++++++--- .../utility/transition_domain_arguments.py | 9 +- ...ess_line_transitiondomain_city_and_more.py | 37 ++ src/registrar/models/transition_domain.py | 35 ++ 5 files changed, 418 insertions(+), 48 deletions(-) create mode 100644 src/registrar/management/commands/load_organization_data.py create mode 100644 src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py new file mode 100644 index 000000000..d5ae51a46 --- /dev/null +++ b/src/registrar/management/commands/load_organization_data.py @@ -0,0 +1,47 @@ +"""Data migration: Send domain invitations once to existing customers.""" + +import argparse +import logging +import copy +import time + +from django.core.management import BaseCommand +from registrar.management.commands.utility.extra_transition_domain_helper import OrganizationDataLoader +from registrar.management.commands.utility.transition_domain_arguments import TransitionDomainArguments +from registrar.models import TransitionDomain +from ...utility.email import send_templated_email, EmailSendingError +from typing import List + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Send domain invitations once to existing customers." + + def add_arguments(self, parser): + """Add command line arguments.""" + + parser.add_argument("--sep", default="|", help="Delimiter character") + + parser.add_argument("--debug", action=argparse.BooleanOptionalAction) + + parser.add_argument("--directory", default="migrationdata", help="Desired directory") + + parser.add_argument( + "--domain_additional_filename", + help="Defines the filename for additional domain data", + required=True, + ) + + parser.add_argument( + "--organization_adhoc_filename", + help="Defines the filename for domain type adhocs", + required=True, + ) + + def handle(self, **options): + """Process the objects in TransitionDomain.""" + args = TransitionDomainArguments(**options) + + load = OrganizationDataLoader(args) + load.update_organization_data_for_all() diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 7961b47c1..640fe9875 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -751,6 +751,257 @@ class FileDataHolder: full_filename = date + "." + filename_without_date return (full_filename, can_infer) +class OrganizationDataLoader: + """Saves organization data onto Transition Domains. Handles file parsing.""" + def __init__(self, options: TransitionDomainArguments): + # Globally stores event logs and organizes them + self.parse_logs = FileTransitionLog() + self.debug = options.debug + + options.pattern_map_params = [ + ( + EnumFilenames.DOMAIN_ADDITIONAL, + options.domain_additional_filename, + DomainAdditionalData, + "domainname", + ), + ( + EnumFilenames.ORGANIZATION_ADHOC, + options.organization_adhoc_filename, + OrganizationAdhoc, + "orgid", + ), + ] + # Reads and parses organization data + self.parsed_data = ExtraTransitionDomain(options) + # options.infer_filenames will always be false when not SETTING.DEBUG + self.parsed_data.parse_all_files(options.infer_filenames) + + self.tds_to_update = [] + self.tds_failed_to_update = [] + + def update_organization_data_for_all(self): + """Updates org data for all TransitionDomains""" + all_transition_domains = TransitionDomain.objects.all() + if len(all_transition_domains) < 1: + logger.error( + f"{TerminalColors.FAIL}" + "No TransitionDomains exist. Cannot update." + f"{TerminalColors.ENDC}" + ) + return None + + # Store all actions we want to perform in tds_to_update + self.prepare_transition_domains(all_transition_domains) + # Then if we don't run into any exceptions, bulk_update it + self.bulk_update_transition_domains(self.tds_to_update) + + def prepare_transition_domains(self, transition_domains): + for item in transition_domains: + try: + updated = self.parse_org_data(item.domain_name, item) + self.tds_to_update.append(updated) + if self.debug: + logger.info(item.display_transition_domain()) + logger.info( + f"{TerminalColors.OKCYAN}" + f"{item.display_transition_domain()}" + f"{TerminalColors.ENDC}" + ) + except Exception as err: + logger.error(err) + self.tds_failed_to_update.append(item) + if self.debug: + logger.error( + f"{TerminalColors.YELLOW}" + f"{item.display_transition_domain()}" + f"{TerminalColors.ENDC}" + ) + + if len(self.tds_failed_to_update) > 0: + logger.error( + "Failed to update. An exception was encountered " + f"on the following TransitionDomains: {[item for item in self.tds_failed_to_update]}" + ) + raise Exception("Failed to update TransitionDomains") + + if not self.debug: + logger.info( + f"Ready to update {len(self.tds_to_update)} TransitionDomains." + ) + else: + logger.info( + f"Ready to update {len(self.tds_to_update)} TransitionDomains: {[item for item in self.tds_failed_to_update]}" + ) + + def bulk_update_transition_domains(self, update_list): + logger.info( + f"{TerminalColors.MAGENTA}" + "Beginning mass TransitionDomain update..." + f"{TerminalColors.ENDC}" + ) + + changed_fields = [ + "address_line", + "city", + "state_territory", + "zipcode", + "country_code", + ] + + TransitionDomain.objects.bulk_update(update_list, changed_fields) + + if not self.debug: + logger.info( + f"{TerminalColors.OKGREEN}" + f"Updated {len(self.tds_to_update)} TransitionDomains." + f"{TerminalColors.ENDC}" + ) + else: + logger.info( + f"{TerminalColors.OKGREEN}" + f"Updated {len(self.tds_to_update)} TransitionDomains: {[item for item in self.tds_failed_to_update]}" + f"{TerminalColors.ENDC}" + ) + + def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: + """Grabs organization_name from the parsed files and associates it + with a transition_domain object, then returns that object.""" + if not isinstance(transition_domain, TransitionDomain): + raise ValueError("Not a valid object, must be TransitionDomain") + + org_info = self.get_org_info(domain_name) + if org_info is None: + self.parse_logs.create_log_item( + EnumFilenames.ORGANIZATION_ADHOC, + LogCode.ERROR, + f"Could not add organization_name on {domain_name}, no data exists.", + domain_name, + not self.debug, + ) + return transition_domain + + # Add street info + transition_domain.address_line = org_info.orgstreet + transition_domain.city = org_info.orgcity + transition_domain.state_territory = org_info.orgstate + transition_domain.zipcode = org_info.orgzip + transition_domain.country_code = org_info.orgcountrycode + + # Log what happened to each field + changed_fields = [ + ("address_line", transition_domain.address_line), + ("city", transition_domain.city), + ("state_territory", transition_domain.state_territory), + ("zipcode", transition_domain.zipcode), + ("country_code", transition_domain.country_code), + ] + self.log_add_or_changed_values(EnumFilenames.AUTHORITY_ADHOC, changed_fields, domain_name) + + return transition_domain + + def get_org_info(self, domain_name) -> OrganizationAdhoc: + """Maps an id given in get_domain_data to a organization_adhoc + record which has its corresponding definition""" + domain_info = self.get_domain_data(domain_name) + if domain_info is None: + return None + org_id = domain_info.orgid + return self.get_organization_adhoc(org_id) + + def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: + """Grabs a corresponding row within the ORGANIZATION_ADHOC file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) + + def get_domain_data(self, desired_id) -> DomainAdditionalData: + """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, + based off a desired_id""" + return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) + + def get_object_by_id(self, file_type: EnumFilenames, desired_id): + """Returns a field in a dictionary based off the type and id. + + vars: + file_type: (constant) EnumFilenames -> Which data file to target. + An example would be `EnumFilenames.DOMAIN_ADHOC`. + + desired_id: str -> Which id you want to search on. + An example would be `"12"` or `"igorville.gov"` + + Explanation: + Each data file has an associated type (file_type) for tracking purposes. + + Each file_type is a dictionary which + contains a dictionary of row[id_field]: object. + + In practice, this would look like: + + EnumFilenames.AUTHORITY_ADHOC: { + "1": AuthorityAdhoc(...), + "2": AuthorityAdhoc(...), + ... + } + + desired_id will then specify which id to grab. If we wanted "1", + then this function will return the value of id "1". + So, `AuthorityAdhoc(...)` + """ + # Grabs a dict associated with the file_type. + # For example, EnumFilenames.DOMAIN_ADDITIONAL. + desired_type = self.parsed_data.file_data.get(file_type) + if desired_type is None: + self.parse_logs.create_log_item( + file_type, + LogCode.ERROR, + f"Type {file_type} does not exist", + ) + return None + + # Grab the value given an Id within that file_type dict. + # For example, "igorville.gov". + obj = desired_type.data.get(desired_id) + if obj is None: + self.parse_logs.create_log_item( + file_type, + LogCode.ERROR, + f"Id {desired_id} does not exist for {file_type.value[0]}", + ) + return obj + + def log_add_or_changed_values(self, file_type, values_to_check, domain_name): + for field_name, value in values_to_check: + str_exists = value is not None and value.strip() != "" + # Logs if we either added to this property, + # or modified it. + self._add_or_change_message( + file_type, + field_name, + value, + domain_name, + str_exists, + ) + + def _add_or_change_message(self, file_type, var_name, changed_value, domain_name, is_update=False): + """Creates a log instance when a property + is successfully changed on a given TransitionDomain.""" + if not is_update: + self.parse_logs.create_log_item( + file_type, + LogCode.INFO, + f"Added {var_name} as '{changed_value}' on {domain_name}", + domain_name, + not self.debug, + ) + else: + self.parse_logs.create_log_item( + file_type, + LogCode.WARNING, + f"Updated existing {var_name} to '{changed_value}' on {domain_name}", + domain_name, + not self.debug, + ) + class ExtraTransitionDomain: """Helper class to aid in storing TransitionDomain data spread across @@ -775,52 +1026,49 @@ class ExtraTransitionDomain: # metadata about each file and associate it with an enum. # That way if we want the data located at the agency_adhoc file, # we can just call EnumFilenames.AGENCY_ADHOC. - pattern_map_params = [ - ( - EnumFilenames.AGENCY_ADHOC, - options.agency_adhoc_filename, - AgencyAdhoc, - "agencyid", - ), - ( - EnumFilenames.DOMAIN_ADDITIONAL, - options.domain_additional_filename, - DomainAdditionalData, - "domainname", - ), - ( - EnumFilenames.DOMAIN_ESCROW, - options.domain_escrow_filename, - DomainEscrow, - "domainname", - ), - ( - EnumFilenames.DOMAIN_ADHOC, - options.domain_adhoc_filename, - DomainTypeAdhoc, - "domaintypeid", - ), - ( - EnumFilenames.ORGANIZATION_ADHOC, - options.organization_adhoc_filename, - OrganizationAdhoc, - "orgid", - ), - ( - EnumFilenames.AUTHORITY_ADHOC, - options.authority_adhoc_filename, - AuthorityAdhoc, - "authorityid", - ), - ( - EnumFilenames.AUTHORITY_ADHOC, - options.authority_adhoc_filename, - AuthorityAdhoc, - "authorityid", - ), - ] + if ( + options.pattern_map_params is None or options.pattern_map_params == [] + ): + options.pattern_map_params = [ + ( + EnumFilenames.AGENCY_ADHOC, + options.agency_adhoc_filename, + AgencyAdhoc, + "agencyid", + ), + ( + EnumFilenames.DOMAIN_ADDITIONAL, + options.domain_additional_filename, + DomainAdditionalData, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ESCROW, + options.domain_escrow_filename, + DomainEscrow, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ADHOC, + options.domain_adhoc_filename, + DomainTypeAdhoc, + "domaintypeid", + ), + ( + EnumFilenames.ORGANIZATION_ADHOC, + options.organization_adhoc_filename, + OrganizationAdhoc, + "orgid", + ), + ( + EnumFilenames.AUTHORITY_ADHOC, + options.authority_adhoc_filename, + AuthorityAdhoc, + "authorityid", + ), + ] - self.file_data = self.populate_file_data(pattern_map_params) + self.file_data = self.populate_file_data(options.pattern_map_params) # TODO - revise comment def populate_file_data(self, pattern_map_params): diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index 56425a7b7..bfe1dd84e 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Optional +from typing import List, Optional from registrar.management.commands.utility.epp_data_containers import EnumFilenames @@ -18,6 +18,9 @@ class TransitionDomainArguments: # Maintains an internal kwargs list and sets values # that match the class definition. def __init__(self, **kwargs): + self.pattern_map_params = [] + if "self.pattern_map_params" in kwargs: + self.pattern_map_params = kwargs["pattern_map_params"] self.kwargs = kwargs for k, v in kwargs.items(): if hasattr(self, k): @@ -36,13 +39,13 @@ class TransitionDomainArguments: limitParse: Optional[int] = field(default=None, repr=True) # Filenames # - # = Adhocs =# + # = Adhocs = # agency_adhoc_filename: Optional[str] = field(default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True) domain_adhoc_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True) organization_adhoc_filename: Optional[str] = field(default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True) authority_adhoc_filename: Optional[str] = field(default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True) - # = Data files =# + # = Data files = # domain_escrow_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True) domain_additional_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True) domain_contacts_filename: Optional[str] = field(default=None, repr=True) diff --git a/src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py b/src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py new file mode 100644 index 000000000..51ce8d6a2 --- /dev/null +++ b/src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.7 on 2023-11-16 19:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0046_transitiondomain_email_transitiondomain_first_name_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="transitiondomain", + name="address_line", + field=models.TextField(blank=True, help_text="Street address", null=True), + ), + migrations.AddField( + model_name="transitiondomain", + name="city", + field=models.TextField(blank=True, help_text="City", null=True), + ), + migrations.AddField( + model_name="transitiondomain", + name="country_code", + field=models.CharField(blank=True, help_text="Country code", max_length=2, null=True), + ), + migrations.AddField( + model_name="transitiondomain", + name="state_territory", + field=models.CharField(blank=True, help_text="State, territory, or military post", max_length=2, null=True), + ), + migrations.AddField( + model_name="transitiondomain", + name="zipcode", + field=models.CharField(blank=True, db_index=True, help_text="Zip code", max_length=10, null=True), + ), + ] diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 3f1c8d641..0f1e0a7bf 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -105,6 +105,36 @@ class TransitionDomain(TimeStampedModel): blank=True, help_text="Phone", ) + address_line = models.TextField( + null=True, + blank=True, + help_text="Street address", + ) + city = models.TextField( + null=True, + blank=True, + help_text="City", + ) + state_territory = models.CharField( + max_length=2, + null=True, + blank=True, + help_text="State, territory, or military post", + ) + zipcode = models.CharField( + max_length=10, + null=True, + blank=True, + help_text="Zip code", + db_index=True, + ) + country_code = models.CharField( + max_length=2, + null=True, + blank=True, + help_text="Country code", + ) + # TODO - Country code? def __str__(self): return f"{self.username}, {self.domain_name}" @@ -128,4 +158,9 @@ class TransitionDomain(TimeStampedModel): f"last_name: {self.last_name}, \n" f"email: {self.email}, \n" f"phone: {self.phone}, \n" + f"address_line: {self.address_line}, \n" + f"city: {self.city}, \n" + f"state_territory: {self.state_territory}, \n" + f"zipcode: {self.zipcode}, \n" + f"country_code: {self.country_code}, \n" ) From 44cd47e51d170814f48473d69f6d28d847a9e680 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:42:20 -0700 Subject: [PATCH 015/119] Add DomainInformation data --- .../commands/load_organization_data.py | 152 +++++++++++++++++- .../utility/extra_transition_domain_helper.py | 4 +- src/registrar/models/transition_domain.py | 1 - 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index d5ae51a46..550f4def9 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -7,8 +7,11 @@ import time from django.core.management import BaseCommand from registrar.management.commands.utility.extra_transition_domain_helper import OrganizationDataLoader +from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from registrar.management.commands.utility.transition_domain_arguments import TransitionDomainArguments from registrar.models import TransitionDomain +from registrar.models.domain import Domain +from registrar.models.domain_information import DomainInformation from ...utility.email import send_templated_email, EmailSendingError from typing import List @@ -43,5 +46,152 @@ class Command(BaseCommand): """Process the objects in TransitionDomain.""" args = TransitionDomainArguments(**options) + changed_fields = [ + "address_line", + "city", + "state_territory", + "zipcode", + "country_code", + ] + proceed = TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + info_to_inspect=f""" + ==Master data file== + domain_additional_filename: {args.domain_additional_filename} + + ==Organization name information== + organization_adhoc_filename: {args.organization_adhoc_filename} + + ==Containing directory== + directory: {args.directory} + + ==Proposed Changes== + For each TransitionDomain, modify the following fields: {changed_fields} + """, + prompt_title="Do you wish to load organization data for TransitionDomains?", + ) + + if not proceed: + return None + + logger.info( + f"{TerminalColors.MAGENTA}" + "Loading organization data onto TransitionDomain tables..." + ) load = OrganizationDataLoader(args) - load.update_organization_data_for_all() + domain_information_to_update = load.update_organization_data_for_all() + + # Reprompt the user to reinspect before updating DomainInformation + proceed = TerminalHelper.prompt_for_execution( + system_exit_on_terminate=True, + info_to_inspect=f""" + ==Master data file== + domain_additional_filename: {args.domain_additional_filename} + + ==Organization name information== + organization_adhoc_filename: {args.organization_adhoc_filename} + + ==Containing directory== + directory: {args.directory} + + ==Proposed Changes== + Number of DomainInformation objects to change: {len(domain_information_to_update)} + """, + prompt_title="Do you wish to load organization data for DomainInformation?", + ) + + if not proceed: + return None + + logger.info( + f"{TerminalColors.MAGENTA}" + "Loading organization data onto DomainInformation tables..." + ) + self.update_domain_information(domain_information_to_update, args.debug) + + def update_domain_information(self, desired_objects: List[TransitionDomain], debug): + di_to_update = [] + di_failed_to_update = [] + for item in desired_objects: + # TODO - this can probably be done in fewer steps + transition_domains = TransitionDomain.objects.filter(username=item.username, domain_name=item.domain_name) + current_transition_domain = self.retrieve_and_assert_single_item(transition_domains, "TransitionDomain", "test") + + domains = Domain.objects.filter(name=current_transition_domain.domain_name) + current_domain = self.retrieve_and_assert_single_item(domains, "Domain", "test") + + domain_informations = DomainInformation.objects.filter(domain=current_domain) + current_domain_information = self.retrieve_and_assert_single_item(domain_informations, "DomainInformation", "test") + + try: + # TODO - add verification to each, for instance check address_line length + current_domain_information.address_line1 = current_transition_domain.address_line + current_domain_information.city = current_transition_domain.city + current_domain_information.state_territory = current_transition_domain.state_territory + current_domain_information.zipcode = current_transition_domain.zipcode + + # TODO - Country Code + #current_domain_information.country_code = current_transition_domain.country_code + except Exception as err: + logger.error(err) + di_failed_to_update.append(current_domain_information) + else: + di_to_update.append(current_domain_information) + + if len(di_failed_to_update) > 0: + logger.error( + "Failed to update. An exception was encountered " + f"on the following TransitionDomains: {[item for item in di_failed_to_update]}" + ) + raise Exception("Failed to update DomainInformations") + + if not debug: + logger.info( + f"Ready to update {len(di_to_update)} TransitionDomains." + ) + else: + logger.info( + f"Ready to update {len(di_to_update)} TransitionDomains: {[item for item in di_to_update]}" + ) + + logger.info( + f"{TerminalColors.MAGENTA}" + "Beginning mass DomainInformation update..." + f"{TerminalColors.ENDC}" + ) + + changed_fields = [ + "address_line1", + "city", + "state_territory", + "zipcode", + #"country_code", + ] + + DomainInformation.objects.bulk_update(di_to_update, changed_fields) + + if not debug: + logger.info( + f"{TerminalColors.OKGREEN}" + f"Updated {len(di_to_update)} DomainInformations." + f"{TerminalColors.ENDC}" + ) + else: + logger.info( + f"{TerminalColors.OKGREEN}" + f"Updated {len(di_to_update)} DomainInformations: {[item for item in di_to_update]}" + f"{TerminalColors.ENDC}" + ) + + # TODO - rename function + update so item_name_for_log works + def retrieve_and_assert_single_item(self, item_queryset, class_name_for_log, item_name_for_log): + """Checks if .filter returns one, and only one, item""" + if item_queryset.count() == 0: + # TODO - custom exception class + raise Exception(f"Could not update. {class_name_for_log} for {item_name_for_log} was not found") + + if item_queryset.count() > 1: + raise Exception(f"Could not update. Duplicate {class_name_for_log} for {item_name_for_log} was found") + + desired_item = item_queryset.get() + return desired_item diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 640fe9875..7643f2d12 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -784,17 +784,17 @@ class OrganizationDataLoader: """Updates org data for all TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() if len(all_transition_domains) < 1: - logger.error( + raise Exception( f"{TerminalColors.FAIL}" "No TransitionDomains exist. Cannot update." f"{TerminalColors.ENDC}" ) - return None # Store all actions we want to perform in tds_to_update self.prepare_transition_domains(all_transition_domains) # Then if we don't run into any exceptions, bulk_update it self.bulk_update_transition_domains(self.tds_to_update) + return self.tds_to_update def prepare_transition_domains(self, transition_domains): for item in transition_domains: diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index 0f1e0a7bf..e8001252d 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -134,7 +134,6 @@ class TransitionDomain(TimeStampedModel): blank=True, help_text="Country code", ) - # TODO - Country code? def __str__(self): return f"{self.username}, {self.domain_name}" From c86a9d5fffe2d1c313e8d0c09dd8254275ad1684 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:16:35 -0700 Subject: [PATCH 016/119] Increase performance --- .../commands/load_organization_data.py | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 550f4def9..9d0c112a5 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -103,35 +103,62 @@ class Command(BaseCommand): if not proceed: return None + if len(domain_information_to_update) == 0: + logger.error( + f"{TerminalColors.MAGENTA}" + "No DomainInformation objects exist" + f"{TerminalColors.ENDC}" + ) + return None + logger.info( f"{TerminalColors.MAGENTA}" - "Loading organization data onto DomainInformation tables..." + "Preparing to load organization data onto DomainInformation tables..." + f"{TerminalColors.ENDC}" ) self.update_domain_information(domain_information_to_update, args.debug) def update_domain_information(self, desired_objects: List[TransitionDomain], debug): di_to_update = [] di_failed_to_update = [] + + # Fetch all TransitionDomains in one query + transition_domains = TransitionDomain.objects.filter( + username__in=[item.username for item in desired_objects], + domain_name__in=[item.domain_name for item in desired_objects] + ) + + # Fetch all Domains in one query + domains = Domain.objects.filter( + name__in=[td.domain_name for td in transition_domains] + ) + + # Fetch all DomainInformations in one query + domain_informations = DomainInformation.objects.filter( + domain__in=domains + ) + + # Create dictionaries for faster lookup + transition_domains_dict = {td.domain_name: td for td in transition_domains} + domains_dict = {d.name: d for d in domains} + domain_informations_dict = {di.domain.name: di for di in domain_informations} + for item in desired_objects: - # TODO - this can probably be done in fewer steps - transition_domains = TransitionDomain.objects.filter(username=item.username, domain_name=item.domain_name) - current_transition_domain = self.retrieve_and_assert_single_item(transition_domains, "TransitionDomain", "test") - - domains = Domain.objects.filter(name=current_transition_domain.domain_name) - current_domain = self.retrieve_and_assert_single_item(domains, "Domain", "test") - - domain_informations = DomainInformation.objects.filter(domain=current_domain) - current_domain_information = self.retrieve_and_assert_single_item(domain_informations, "DomainInformation", "test") - try: + current_transition_domain = transition_domains_dict[item.domain_name] + current_domain = domains_dict[current_transition_domain.domain_name] + current_domain_information = domain_informations_dict[current_domain.name] + # TODO - add verification to each, for instance check address_line length current_domain_information.address_line1 = current_transition_domain.address_line current_domain_information.city = current_transition_domain.city current_domain_information.state_territory = current_transition_domain.state_territory current_domain_information.zipcode = current_transition_domain.zipcode - # TODO - Country Code #current_domain_information.country_code = current_transition_domain.country_code + + if debug: + logger.info(f"Updating {current_domain.name}...") except Exception as err: logger.error(err) di_failed_to_update.append(current_domain_information) @@ -182,16 +209,3 @@ class Command(BaseCommand): f"Updated {len(di_to_update)} DomainInformations: {[item for item in di_to_update]}" f"{TerminalColors.ENDC}" ) - - # TODO - rename function + update so item_name_for_log works - def retrieve_and_assert_single_item(self, item_queryset, class_name_for_log, item_name_for_log): - """Checks if .filter returns one, and only one, item""" - if item_queryset.count() == 0: - # TODO - custom exception class - raise Exception(f"Could not update. {class_name_for_log} for {item_name_for_log} was not found") - - if item_queryset.count() > 1: - raise Exception(f"Could not update. Duplicate {class_name_for_log} for {item_name_for_log} was found") - - desired_item = item_queryset.get() - return desired_item From d109f5a265b0939a8bbcc9a02a881720ca2980ed Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 17 Nov 2023 06:31:46 -0500 Subject: [PATCH 017/119] added comment --- src/registrar/forms/domain.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 1ef6ce354..fb1de8d8d 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -233,6 +233,12 @@ class DomainDsdataForm(forms.Form): """Form for adding or editing DNSSEC DS Data to a domain.""" def validate_hexadecimal(value): + """ + Tests that string matches all hexadecimal values. + + Raise validation error to display error in form + if invalid caracters entered + """ if not re.match(r"^[0-9a-fA-F]+$", value): raise forms.ValidationError(str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_CHARS))) From 464279bf3374580898e4efd358f1ab108bce458b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 17 Nov 2023 06:35:44 -0500 Subject: [PATCH 018/119] formatted for linter --- src/registrar/forms/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index fb1de8d8d..d15248c10 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -235,7 +235,7 @@ class DomainDsdataForm(forms.Form): def validate_hexadecimal(value): """ Tests that string matches all hexadecimal values. - + Raise validation error to display error in form if invalid caracters entered """ From 7b6929c176a647025e65e049d410abdd28c5d4b7 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Fri, 17 Nov 2023 07:37:31 -0500 Subject: [PATCH 019/119] getter from registry also updates registrar --- src/registrar/models/domain.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 94924a698..780c61435 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -211,8 +211,16 @@ class Domain(TimeStampedModel, DomainHelper): @Cache def registry_expiration_date(self) -> date: - """Get or set the `ex_date` element from the registry.""" - return self._get_property("ex_date") + """Get or set the `ex_date` element from the registry. + Additionally, update the expiration date in the registrar""" + try: + self.expiration_date = self._get_property("ex_date") + self.save() + return self.expiration_date + except Exception as e: + # exception raised during the save to registrar + logger.error(f"error updating expiration date in registrar: {e}") + raise (e) @registry_expiration_date.setter # type: ignore def registry_expiration_date(self, ex_date: date): From b651fbb72d66b6b1a8b0489631e5a9abee634831 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Fri, 17 Nov 2023 07:34:53 -0800 Subject: [PATCH 020/119] updated the bug fix instructions --- docs/operations/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/operations/README.md b/docs/operations/README.md index f18d24115..4c7f182bd 100644 --- a/docs/operations/README.md +++ b/docs/operations/README.md @@ -54,9 +54,10 @@ If a bug fix or feature needs to be made to stable out of the normal cycle, this In the case where a bug fix or feature needs to be added outside of the normal cycle, the code-fix branch and release will be handled differently than normal: 1. Code will need to be branched NOT off of main, but off of the same commit as the most recent stable commit. This should be the one tagged with the most recent vX.XX.XX value. -2. After making the bug fix, the approved PR will branch will be tagged with a new release tag, incrementing the patch value from the last commit number. -3. This branch then needs to be merged to main per the usual process. -4. This same branch should be merged into staging. +2. After making the bug fix, the approved PR branch will not be merged yet, instead it will be tagged with a new release tag, incrementing the patch value from the last commit number. +3. If main and stable are on the the same commit then merge this branch into the staging using the staging release tag (staging-). +4. If staging is already ahead stable, you may need to create another branch that is based off of the current staging commit, merge in your code change and then tag that branch with the staging release. +5. Wait to merge your original branch until both deploys finish. Once they succeed then merge to main per the usual process. ## Serving static assets We are using [WhiteNoise](http://whitenoise.evans.io/en/stable/index.html) plugin to serve our static assets on cloud.gov. This plugin is added to the `MIDDLEWARE` list in our apps `settings.py`. From 13b1ca02387ec09fc27782b94ebb7c55d6f0d9a0 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 10:04:09 -0700 Subject: [PATCH 021/119] Remove country code --- .../commands/load_organization_data.py | 35 ++++++++++--------- .../utility/extra_transition_domain_helper.py | 8 ++--- ...ess_line_transitiondomain_city_and_more.py | 5 --- src/registrar/models/transition_domain.py | 7 ---- 4 files changed, 23 insertions(+), 32 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 9d0c112a5..937286a07 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -51,7 +51,6 @@ class Command(BaseCommand): "city", "state_territory", "zipcode", - "country_code", ] proceed = TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, @@ -122,43 +121,48 @@ class Command(BaseCommand): di_to_update = [] di_failed_to_update = [] - # Fetch all TransitionDomains in one query + # Grab each TransitionDomain we want to change. Store it. + # Fetches all TransitionDomains in one query. transition_domains = TransitionDomain.objects.filter( username__in=[item.username for item in desired_objects], domain_name__in=[item.domain_name for item in desired_objects] ) - # Fetch all Domains in one query + if len(desired_objects) != len(transition_domains): + raise Exception("Could not find all desired TransitionDomains") + + # Then, for each domain_name grab the associated domain object. + # Fetches all Domains in one query. domains = Domain.objects.filter( name__in=[td.domain_name for td in transition_domains] ) - # Fetch all DomainInformations in one query + # Then, use each domain object to map domain <--> DomainInformation + # Fetches all DomainInformations in one query. domain_informations = DomainInformation.objects.filter( domain__in=domains ) # Create dictionaries for faster lookup - transition_domains_dict = {td.domain_name: td for td in transition_domains} domains_dict = {d.name: d for d in domains} domain_informations_dict = {di.domain.name: di for di in domain_informations} - for item in desired_objects: + for item in transition_domains: try: - current_transition_domain = transition_domains_dict[item.domain_name] - current_domain = domains_dict[current_transition_domain.domain_name] + # Grab the current Domain. This ensures we are pointing towards the right place. + current_domain = domains_dict[item.domain_name] + + # Based on the current domain, grab the right DomainInformation object. current_domain_information = domain_informations_dict[current_domain.name] - # TODO - add verification to each, for instance check address_line length - current_domain_information.address_line1 = current_transition_domain.address_line - current_domain_information.city = current_transition_domain.city - current_domain_information.state_territory = current_transition_domain.state_territory - current_domain_information.zipcode = current_transition_domain.zipcode - # TODO - Country Code - #current_domain_information.country_code = current_transition_domain.country_code + current_domain_information.address_line1 = item.address_line + current_domain_information.city = item.city + current_domain_information.state_territory = item.state_territory + current_domain_information.zipcode = item.zipcode if debug: logger.info(f"Updating {current_domain.name}...") + except Exception as err: logger.error(err) di_failed_to_update.append(current_domain_information) @@ -192,7 +196,6 @@ class Command(BaseCommand): "city", "state_territory", "zipcode", - #"country_code", ] DomainInformation.objects.bulk_update(di_to_update, changed_fields) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 7643f2d12..96cc550b3 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -804,6 +804,7 @@ class OrganizationDataLoader: if self.debug: logger.info(item.display_transition_domain()) logger.info( + f"Successfully updated TransitionDomain: \n" f"{TerminalColors.OKCYAN}" f"{item.display_transition_domain()}" f"{TerminalColors.ENDC}" @@ -813,6 +814,7 @@ class OrganizationDataLoader: self.tds_failed_to_update.append(item) if self.debug: logger.error( + f"Failed to update TransitionDomain: \n" f"{TerminalColors.YELLOW}" f"{item.display_transition_domain()}" f"{TerminalColors.ENDC}" @@ -846,7 +848,6 @@ class OrganizationDataLoader: "city", "state_territory", "zipcode", - "country_code", ] TransitionDomain.objects.bulk_update(update_list, changed_fields) @@ -886,15 +887,14 @@ class OrganizationDataLoader: transition_domain.city = org_info.orgcity transition_domain.state_territory = org_info.orgstate transition_domain.zipcode = org_info.orgzip - transition_domain.country_code = org_info.orgcountrycode - # Log what happened to each field + # Log what happened to each field. The first value + # is the field name that was updated, second is the value changed_fields = [ ("address_line", transition_domain.address_line), ("city", transition_domain.city), ("state_territory", transition_domain.state_territory), ("zipcode", transition_domain.zipcode), - ("country_code", transition_domain.country_code), ] self.log_add_or_changed_values(EnumFilenames.AUTHORITY_ADHOC, changed_fields, domain_name) diff --git a/src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py b/src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py index 51ce8d6a2..a312f62f2 100644 --- a/src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py +++ b/src/registrar/migrations/0047_transitiondomain_address_line_transitiondomain_city_and_more.py @@ -19,11 +19,6 @@ class Migration(migrations.Migration): name="city", field=models.TextField(blank=True, help_text="City", null=True), ), - migrations.AddField( - model_name="transitiondomain", - name="country_code", - field=models.CharField(blank=True, help_text="Country code", max_length=2, null=True), - ), migrations.AddField( model_name="transitiondomain", name="state_territory", diff --git a/src/registrar/models/transition_domain.py b/src/registrar/models/transition_domain.py index e8001252d..c5b9b125c 100644 --- a/src/registrar/models/transition_domain.py +++ b/src/registrar/models/transition_domain.py @@ -128,12 +128,6 @@ class TransitionDomain(TimeStampedModel): help_text="Zip code", db_index=True, ) - country_code = models.CharField( - max_length=2, - null=True, - blank=True, - help_text="Country code", - ) def __str__(self): return f"{self.username}, {self.domain_name}" @@ -161,5 +155,4 @@ class TransitionDomain(TimeStampedModel): f"city: {self.city}, \n" f"state_territory: {self.state_territory}, \n" f"zipcode: {self.zipcode}, \n" - f"country_code: {self.country_code}, \n" ) From a6db4b7145de14616a2dfcd8e6eb44c9c2bcf821 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:26:46 -0700 Subject: [PATCH 022/119] Data chunking / don't update existing data --- .../commands/load_organization_data.py | 75 +++++++++++++++---- .../utility/extra_transition_domain_helper.py | 10 ++- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 937286a07..2c07bbb0a 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -2,8 +2,6 @@ import argparse import logging -import copy -import time from django.core.management import BaseCommand from registrar.management.commands.utility.extra_transition_domain_helper import OrganizationDataLoader @@ -12,7 +10,7 @@ from registrar.management.commands.utility.transition_domain_arguments import Tr from registrar.models import TransitionDomain from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation -from ...utility.email import send_templated_email, EmailSendingError +from django.core.paginator import Paginator from typing import List logger = logging.getLogger(__name__) @@ -120,6 +118,9 @@ class Command(BaseCommand): def update_domain_information(self, desired_objects: List[TransitionDomain], debug): di_to_update = [] di_failed_to_update = [] + # These are fields that we COULD update, but fields we choose not to update. + # For instance, if the user already entered data - lets not corrupt that. + di_skipped = [] # Grab each TransitionDomain we want to change. Store it. # Fetches all TransitionDomains in one query. @@ -137,9 +138,27 @@ class Command(BaseCommand): name__in=[td.domain_name for td in transition_domains] ) + + # Start with all DomainInformation objects + filtered_domain_informations = DomainInformation.objects.all() + + changed_fields = [ + "address_line1", + "city", + "state_territory", + "zipcode", + ] + + # Chain filter calls for each field. This checks to see if the end user + # made a change to ANY field in changed_fields. If they did, don't update their information. + # We assume that if they made a change, we don't want to interfere with that. + for field in changed_fields: + # For each changed_field, check if no data exists + filtered_domain_informations = filtered_domain_informations.filter(**{f'{field}__isnull': True}) + # Then, use each domain object to map domain <--> DomainInformation # Fetches all DomainInformations in one query. - domain_informations = DomainInformation.objects.filter( + domain_informations = filtered_domain_informations.filter( domain__in=domains ) @@ -149,32 +168,52 @@ class Command(BaseCommand): for item in transition_domains: try: + should_update = True # Grab the current Domain. This ensures we are pointing towards the right place. current_domain = domains_dict[item.domain_name] # Based on the current domain, grab the right DomainInformation object. - current_domain_information = domain_informations_dict[current_domain.name] + if current_domain.name in domain_informations_dict: + current_domain_information = domain_informations_dict[current_domain.name] + current_domain_information.address_line1 = item.address_line + current_domain_information.city = item.city + current_domain_information.state_territory = item.state_territory + current_domain_information.zipcode = item.zipcode + + if debug: + logger.info(f"Updating {current_domain.name}...") - current_domain_information.address_line1 = item.address_line - current_domain_information.city = item.city - current_domain_information.state_territory = item.state_territory - current_domain_information.zipcode = item.zipcode - - if debug: - logger.info(f"Updating {current_domain.name}...") + else: + logger.info( + f"{TerminalColors.YELLOW}" + f"Domain {current_domain.name} was updated by a user. Cannot update." + f"{TerminalColors.ENDC}" + ) + should_update = False except Exception as err: logger.error(err) - di_failed_to_update.append(current_domain_information) + di_failed_to_update.append(item) else: - di_to_update.append(current_domain_information) + if should_update: + di_to_update.append(current_domain_information) + else: + # TODO either update to name for all, + # or have this filter to the right field + di_skipped.append(item) if len(di_failed_to_update) > 0: logger.error( + f"{TerminalColors.FAIL}" "Failed to update. An exception was encountered " f"on the following TransitionDomains: {[item for item in di_failed_to_update]}" + f"{TerminalColors.ENDC}" ) raise Exception("Failed to update DomainInformations") + + skipped_count = len(di_skipped) + if skipped_count > 0: + logger.info(f"Skipped updating {skipped_count} fields. User-supplied data exists") if not debug: logger.info( @@ -198,7 +237,13 @@ class Command(BaseCommand): "zipcode", ] - DomainInformation.objects.bulk_update(di_to_update, changed_fields) + batch_size = 1000 + # Create a Paginator object. Bulk_update on the full dataset + # is too memory intensive for our current app config, so we can chunk this data instead. + paginator = Paginator(di_to_update, batch_size) + for page_num in paginator.page_range: + page = paginator.page(page_num) + DomainInformation.objects.bulk_update(page.object_list, changed_fields) if not debug: logger.info( diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 96cc550b3..be84e7681 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -10,7 +10,7 @@ import logging import os import sys from typing import Dict - +from django.core.paginator import Paginator from registrar.models.transition_domain import TransitionDomain from .epp_data_containers import ( @@ -850,7 +850,13 @@ class OrganizationDataLoader: "zipcode", ] - TransitionDomain.objects.bulk_update(update_list, changed_fields) + batch_size = 1000 + # Create a Paginator object. Bulk_update on the full dataset + # is too memory intensive for our current app config, so we can chunk this data instead. + paginator = Paginator(update_list, batch_size) + for page_num in paginator.page_range: + page = paginator.page(page_num) + TransitionDomain.objects.bulk_update(page.object_list, changed_fields) if not self.debug: logger.info( From 9e7f111206ad1e5510fc010517965c2354abeceb Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:43:40 -0700 Subject: [PATCH 023/119] Update load_organization_data.py --- src/registrar/management/commands/load_organization_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 2c07bbb0a..a4666d461 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -127,7 +127,7 @@ class Command(BaseCommand): transition_domains = TransitionDomain.objects.filter( username__in=[item.username for item in desired_objects], domain_name__in=[item.domain_name for item in desired_objects] - ) + ).distinct() if len(desired_objects) != len(transition_domains): raise Exception("Could not find all desired TransitionDomains") From 85ee97d615057ec31c646f785845a6810e142c50 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:02:29 -0700 Subject: [PATCH 024/119] Add test cases One is still breaking, not mine but a related test. Test interference issue? --- .../commands/load_organization_data.py | 40 +++++- .../test_transition_domain_migrations.py | 125 ++++++++++++++++++ 2 files changed, 162 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index a4666d461..5d9d70716 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -1,6 +1,7 @@ """Data migration: Send domain invitations once to existing customers.""" import argparse +import json import logging from django.core.management import BaseCommand @@ -22,6 +23,11 @@ class Command(BaseCommand): def add_arguments(self, parser): """Add command line arguments.""" + parser.add_argument( + "migration_json_filename", + help=("A JSON file that holds the location and filenames" "of all the data files used for migrations"), + ) + parser.add_argument("--sep", default="|", help="Delimiter character") parser.add_argument("--debug", action=argparse.BooleanOptionalAction) @@ -31,17 +37,45 @@ class Command(BaseCommand): parser.add_argument( "--domain_additional_filename", help="Defines the filename for additional domain data", - required=True, ) parser.add_argument( "--organization_adhoc_filename", help="Defines the filename for domain type adhocs", - required=True, ) - def handle(self, **options): + def handle(self, migration_json_filename, **options): """Process the objects in TransitionDomain.""" + + # === Parse JSON file === # + # Desired directory for additional TransitionDomain data + # (In the event they are stored seperately) + directory = options["directory"] + # Add a slash if the last character isn't one + if directory and directory[-1] != "/": + directory += "/" + + json_filepath = directory + migration_json_filename + + # If a JSON was provided, use its values instead of defaults. + with open(json_filepath, "r") as jsonFile: + # load JSON object as a dictionary + try: + data = json.load(jsonFile) + # Create an instance of TransitionDomainArguments + # Iterate over the data from the JSON file + for key, value in data.items(): + if value is not None and value.strip() != "": + options[key] = value + except Exception as err: + logger.error( + f"{TerminalColors.FAIL}" + "There was an error loading " + "the JSON responsible for providing filepaths." + f"{TerminalColors.ENDC}" + ) + raise err + # === End parse JSON file === # args = TransitionDomainArguments(**options) changed_fields = [ diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index c6418f013..0c959673d 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -81,6 +81,19 @@ class TestMigrations(TestCase): migrationJSON=self.migration_json_filename, disablePrompts=True, ) + + def run_load_organization_data(self): + # noqa here (E501) because splitting this up makes it + # confusing to read. + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command( + "load_organization_data", + self.migration_json_filename, + directory=self.test_data_file_location, + ) def compare_tables( self, @@ -157,6 +170,118 @@ class TestMigrations(TestCase): self.assertEqual(total_domain_informations, expected_total_domain_informations) self.assertEqual(total_domain_invitations, expected_total_domain_invitations) + def test_load_organization_data_transition_domain(self): + self.maxDiff = None + # == First, parse all existing data == # + self.run_master_script() + + # == Second, try adding org data to it == # + self.run_load_organization_data() + + # == Third, test that we've loaded data as we expect == # + transition_domains = TransitionDomain.objects.filter(domain_name="fakewebsite2.gov") + + # Should return three objects (three unique emails) + self.assertEqual(transition_domains.count(), 3) + + # Lets test the first one + transition = transition_domains.first() + expected_transition_domain = TransitionDomain( + id=6, + username='alexandra.bobbitt5@test.com', + domain_name='fakewebsite2.gov', + status='on hold', + email_sent=True, + organization_type='Federal', + organization_name='Fanoodle', + federal_type='Executive', + federal_agency='Department of Commerce', + epp_creation_date=datetime.date(2004, 5, 7), + epp_expiration_date=datetime.date(2023, 9, 30), + first_name='Seline', + middle_name='testmiddle2', + last_name='Tower', + title=None, + email='stower3@answers.com', + phone='151-539-6028', + address_line='93001 Arizona Drive', + city='Columbus', + state_territory='Oh', + zipcode='43268' + ) + + self.assertEqual(transition, expected_transition_domain) + + def test_load_organization_data_domain_information(self): + self.maxDiff = None + # == First, parse all existing data == # + self.run_master_script() + + # == Second, try adding org data to it == # + self.run_load_organization_data() + + # == Third, test that we've loaded data as we expect == # + _domain = Domain.objects.filter(name="fakewebsite2.gov").get() + domain_information = DomainInformation.objects.filter(domain=_domain).get() + expected_domain_information = DomainInformation( + id=4, + creator_id=1, + domain_application_id=None, + organization_type='federal', + federally_recognized_tribe=None, + state_recognized_tribe=None, + tribe_name=None, + federal_agency='Department of Commerce', + federal_type='executive', + is_election_board=None, + organization_name='Fanoodle', + address_line1='93001 Arizona Drive', + address_line2=None, + city='Columbus', + state_territory='Oh', + zipcode='43268', + urbanization=None, + about_your_organization=None, + authorizing_official_id=5, + domain_id=4, + submitter_id=None, + purpose=None, + no_other_contacts_rationale=None, + anything_else=None, + is_policy_acknowledged=None + ) + self.assertEqual(domain_information, expected_domain_information) + + def test_load_organization_data_integrity(self): + """Validates data integrity with the load_org_data command""" + # First, parse all existing data + self.run_master_script() + + # Second, try adding org data to it + self.run_load_organization_data() + + # Third, test that we didn't corrupt any data + expected_total_transition_domains = 9 + expected_total_domains = 5 + expected_total_domain_informations = 5 + expected_total_domain_invitations = 8 + + expected_missing_domains = 0 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 0 + # we expect 1 missing invite from anomaly.gov (an injected error) + expected_missing_domain_invitations = 1 + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) + def test_master_migration_functions(self): """Run the full master migration script using local test data. NOTE: This is more of an integration test and so far does not From f1acd46588ee81227de69b8021b42de512522c88 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:04:06 -0700 Subject: [PATCH 025/119] Fix test cases --- .../commands/load_organization_data.py | 4 + .../commands/send_domain_invitations.py | 12 +- .../test_transition_domain_migrations.py | 363 ++++++++++++------ 3 files changed, 263 insertions(+), 116 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 5d9d70716..95743c6b8 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -204,6 +204,10 @@ class Command(BaseCommand): try: should_update = True # Grab the current Domain. This ensures we are pointing towards the right place. + if item.domain_name not in domains_dict: + logger.error(f"Could not add {item.domain_name}. Domain does not exist.") + di_failed_to_update.append(item) + continue current_domain = domains_dict[item.domain_name] # Based on the current domain, grab the right DomainInformation object. diff --git a/src/registrar/management/commands/send_domain_invitations.py b/src/registrar/management/commands/send_domain_invitations.py index 603fbce3a..0f8ca1c46 100644 --- a/src/registrar/management/commands/send_domain_invitations.py +++ b/src/registrar/management/commands/send_domain_invitations.py @@ -152,6 +152,12 @@ class Command(BaseCommand): for domain_name in email_data["domains"]: # self.transition_domains is a queryset so we can sub-select # from it and use the objects to mark them as sent - this_transition_domain = self.transition_domains.get(username=this_email, domain_name=domain_name) - this_transition_domain.email_sent = True - this_transition_domain.save() + transition_domains = self.transition_domains.filter(username=this_email, domain_name=domain_name) + if len(transition_domains) == 1: + this_transition_domain = transition_domains.get() + this_transition_domain.email_sent = True + this_transition_domain.save() + elif len(transition_domains) > 1: + logger.error(f"Multiple TransitionDomains exist for {this_email}") + else: + logger.error(f"No TransitionDomain exists for {this_email}") diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 0c959673d..d5183837a 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -18,6 +18,254 @@ from unittest.mock import patch from .common import less_console_noise +class TestOrganizationMigration(TestCase): + def setUp(self): + """ """ + # self.load_transition_domain_script = "load_transition_domain", + # self.transfer_script = "transfer_transition_domains_to_domains", + # self.master_script = "load_transition_domain", + + self.test_data_file_location = "registrar/tests/data" + self.test_domain_contact_filename = "test_domain_contacts.txt" + self.test_contact_filename = "test_contacts.txt" + self.test_domain_status_filename = "test_domain_statuses.txt" + + # Files for parsing additional TransitionDomain data + self.test_agency_adhoc_filename = "test_agency_adhoc.txt" + self.test_authority_adhoc_filename = "test_authority_adhoc.txt" + self.test_domain_additional = "test_domain_additional.txt" + self.test_domain_types_adhoc = "test_domain_types_adhoc.txt" + self.test_escrow_domains_daily = "test_escrow_domains_daily" + self.test_organization_adhoc = "test_organization_adhoc.txt" + self.migration_json_filename = "test_migrationFilepaths.json" + + def tearDown(self): + # Delete domain information + Domain.objects.all().delete() + DomainInformation.objects.all().delete() + DomainInvitation.objects.all().delete() + TransitionDomain.objects.all().delete() + + # Delete users + User.objects.all().delete() + UserDomainRole.objects.all().delete() + + def run_load_domains(self): + # noqa here because splitting this up makes it confusing. + # ES501 + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command( + "load_transition_domain", + self.migration_json_filename, + directory=self.test_data_file_location, + ) + + def run_transfer_domains(self): + call_command("transfer_transition_domains_to_domains") + + def run_load_organization_data(self): + # noqa here (E501) because splitting this up makes it + # confusing to read. + with patch( + "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa + return_value=True, + ): + call_command( + "load_organization_data", + self.migration_json_filename, + directory=self.test_data_file_location, + ) + + def compare_tables( + self, + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ): + """Does a diff between the transition_domain and the following tables: + domain, domain_information and the domain_invitation. + Verifies that the data loaded correctly.""" + + missing_domains = [] + duplicate_domains = [] + missing_domain_informations = [] + missing_domain_invites = [] + for transition_domain in TransitionDomain.objects.all(): # DEBUG: + transition_domain_name = transition_domain.domain_name + transition_domain_email = transition_domain.username + + # Check Domain table + matching_domains = Domain.objects.filter(name=transition_domain_name) + # Check Domain Information table + matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) + # Check Domain Invitation table + matching_domain_invitations = DomainInvitation.objects.filter( + email=transition_domain_email.lower(), + domain__name=transition_domain_name, + ) + + if len(matching_domains) == 0: + missing_domains.append(transition_domain_name) + elif len(matching_domains) > 1: + duplicate_domains.append(transition_domain_name) + if len(matching_domain_informations) == 0: + missing_domain_informations.append(transition_domain_name) + if len(matching_domain_invitations) == 0: + missing_domain_invites.append(transition_domain_name) + + total_missing_domains = len(missing_domains) + total_duplicate_domains = len(duplicate_domains) + total_missing_domain_informations = len(missing_domain_informations) + total_missing_domain_invitations = len(missing_domain_invites) + + total_transition_domains = len(TransitionDomain.objects.all()) + total_domains = len(Domain.objects.all()) + total_domain_informations = len(DomainInformation.objects.all()) + total_domain_invitations = len(DomainInvitation.objects.all()) + + print( + f""" + total_missing_domains = {len(missing_domains)} + total_duplicate_domains = {len(duplicate_domains)} + total_missing_domain_informations = {len(missing_domain_informations)} + total_missing_domain_invitations = {total_missing_domain_invitations} + + total_transition_domains = {len(TransitionDomain.objects.all())} + total_domains = {len(Domain.objects.all())} + total_domain_informations = {len(DomainInformation.objects.all())} + total_domain_invitations = {len(DomainInvitation.objects.all())} + """ + ) + self.assertEqual(total_missing_domains, expected_missing_domains) + self.assertEqual(total_duplicate_domains, expected_duplicate_domains) + self.assertEqual(total_missing_domain_informations, expected_missing_domain_informations) + self.assertEqual(total_missing_domain_invitations, expected_missing_domain_invitations) + + self.assertEqual(total_transition_domains, expected_total_transition_domains) + self.assertEqual(total_domains, expected_total_domains) + self.assertEqual(total_domain_informations, expected_total_domain_informations) + self.assertEqual(total_domain_invitations, expected_total_domain_invitations) + + def test_load_organization_data_transition_domain(self): + # == First, parse all existing data == # + self.run_load_domains() + self.run_transfer_domains() + + # == Second, try adding org data to it == # + self.run_load_organization_data() + + # == Third, test that we've loaded data as we expect == # + transition_domains = TransitionDomain.objects.filter(domain_name="fakewebsite2.gov") + + # Should return three objects (three unique emails) + self.assertEqual(transition_domains.count(), 3) + + # Lets test the first one + transition = transition_domains.first() + expected_transition_domain = TransitionDomain( + id=24, + username='alexandra.bobbitt5@test.com', + domain_name='fakewebsite2.gov', + status='on hold', + email_sent=True, + organization_type='Federal', + organization_name='Fanoodle', + federal_type='Executive', + federal_agency='Department of Commerce', + epp_creation_date=datetime.date(2004, 5, 7), + epp_expiration_date=datetime.date(2023, 9, 30), + first_name='Seline', + middle_name='testmiddle2', + last_name='Tower', + title=None, + email='stower3@answers.com', + phone='151-539-6028', + address_line='93001 Arizona Drive', + city='Columbus', + state_territory='Oh', + zipcode='43268' + ) + + self.assertEqual(transition, expected_transition_domain) + + def test_load_organization_data_domain_information(self): + # == First, parse all existing data == # + self.run_load_domains() + self.run_transfer_domains() + + # == Second, try adding org data to it == # + self.run_load_organization_data() + + # == Third, test that we've loaded data as we expect == # + _domain = Domain.objects.filter(name="fakewebsite2.gov").get() + domain_information = DomainInformation.objects.filter(domain=_domain).get() + expected_domain_information = DomainInformation( + id=4, + creator_id=1, + domain_application_id=None, + organization_type='federal', + federally_recognized_tribe=None, + state_recognized_tribe=None, + tribe_name=None, + federal_agency='Department of Commerce', + federal_type='executive', + is_election_board=None, + organization_name='Fanoodle', + address_line1='93001 Arizona Drive', + address_line2=None, + city='Columbus', + state_territory='Oh', + zipcode='43268', + urbanization=None, + about_your_organization=None, + authorizing_official_id=5, + domain_id=4, + submitter_id=None, + purpose=None, + no_other_contacts_rationale=None, + anything_else=None, + is_policy_acknowledged=None + ) + self.assertEqual(domain_information, expected_domain_information) + + def test_load_organization_data_integrity(self): + """Validates data integrity with the load_org_data command""" + # First, parse all existing data + self.run_load_domains() + self.run_transfer_domains() + + # Second, try adding org data to it + self.run_load_organization_data() + + # Third, test that we didn't corrupt any data + expected_total_transition_domains = 9 + expected_total_domains = 5 + expected_total_domain_informations = 5 + expected_total_domain_invitations = 8 + + expected_missing_domains = 0 + expected_duplicate_domains = 0 + expected_missing_domain_informations = 0 + # we expect 1 missing invite from anomaly.gov (an injected error) + expected_missing_domain_invitations = 1 + self.compare_tables( + expected_total_transition_domains, + expected_total_domains, + expected_total_domain_informations, + expected_total_domain_invitations, + expected_missing_domains, + expected_duplicate_domains, + expected_missing_domain_informations, + expected_missing_domain_invitations, + ) class TestMigrations(TestCase): def setUp(self): @@ -41,11 +289,12 @@ class TestMigrations(TestCase): self.migration_json_filename = "test_migrationFilepaths.json" def tearDown(self): + super().tearDown() # Delete domain information TransitionDomain.objects.all().delete() Domain.objects.all().delete() - DomainInvitation.objects.all().delete() DomainInformation.objects.all().delete() + DomainInvitation.objects.all().delete() # Delete users User.objects.all().delete() @@ -170,118 +419,6 @@ class TestMigrations(TestCase): self.assertEqual(total_domain_informations, expected_total_domain_informations) self.assertEqual(total_domain_invitations, expected_total_domain_invitations) - def test_load_organization_data_transition_domain(self): - self.maxDiff = None - # == First, parse all existing data == # - self.run_master_script() - - # == Second, try adding org data to it == # - self.run_load_organization_data() - - # == Third, test that we've loaded data as we expect == # - transition_domains = TransitionDomain.objects.filter(domain_name="fakewebsite2.gov") - - # Should return three objects (three unique emails) - self.assertEqual(transition_domains.count(), 3) - - # Lets test the first one - transition = transition_domains.first() - expected_transition_domain = TransitionDomain( - id=6, - username='alexandra.bobbitt5@test.com', - domain_name='fakewebsite2.gov', - status='on hold', - email_sent=True, - organization_type='Federal', - organization_name='Fanoodle', - federal_type='Executive', - federal_agency='Department of Commerce', - epp_creation_date=datetime.date(2004, 5, 7), - epp_expiration_date=datetime.date(2023, 9, 30), - first_name='Seline', - middle_name='testmiddle2', - last_name='Tower', - title=None, - email='stower3@answers.com', - phone='151-539-6028', - address_line='93001 Arizona Drive', - city='Columbus', - state_territory='Oh', - zipcode='43268' - ) - - self.assertEqual(transition, expected_transition_domain) - - def test_load_organization_data_domain_information(self): - self.maxDiff = None - # == First, parse all existing data == # - self.run_master_script() - - # == Second, try adding org data to it == # - self.run_load_organization_data() - - # == Third, test that we've loaded data as we expect == # - _domain = Domain.objects.filter(name="fakewebsite2.gov").get() - domain_information = DomainInformation.objects.filter(domain=_domain).get() - expected_domain_information = DomainInformation( - id=4, - creator_id=1, - domain_application_id=None, - organization_type='federal', - federally_recognized_tribe=None, - state_recognized_tribe=None, - tribe_name=None, - federal_agency='Department of Commerce', - federal_type='executive', - is_election_board=None, - organization_name='Fanoodle', - address_line1='93001 Arizona Drive', - address_line2=None, - city='Columbus', - state_territory='Oh', - zipcode='43268', - urbanization=None, - about_your_organization=None, - authorizing_official_id=5, - domain_id=4, - submitter_id=None, - purpose=None, - no_other_contacts_rationale=None, - anything_else=None, - is_policy_acknowledged=None - ) - self.assertEqual(domain_information, expected_domain_information) - - def test_load_organization_data_integrity(self): - """Validates data integrity with the load_org_data command""" - # First, parse all existing data - self.run_master_script() - - # Second, try adding org data to it - self.run_load_organization_data() - - # Third, test that we didn't corrupt any data - expected_total_transition_domains = 9 - expected_total_domains = 5 - expected_total_domain_informations = 5 - expected_total_domain_invitations = 8 - - expected_missing_domains = 0 - expected_duplicate_domains = 0 - expected_missing_domain_informations = 0 - # we expect 1 missing invite from anomaly.gov (an injected error) - expected_missing_domain_invitations = 1 - self.compare_tables( - expected_total_transition_domains, - expected_total_domains, - expected_total_domain_informations, - expected_total_domain_invitations, - expected_missing_domains, - expected_duplicate_domains, - expected_missing_domain_informations, - expected_missing_domain_invitations, - ) - def test_master_migration_functions(self): """Run the full master migration script using local test data. NOTE: This is more of an integration test and so far does not From 8ec24bfb0d4049316221a6129fae38422cbbcf1b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:09:47 -0700 Subject: [PATCH 026/119] Test --- src/registrar/tests/test_transition_domain_migrations.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index d5183837a..9ae9a66be 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -171,7 +171,6 @@ class TestOrganizationMigration(TestCase): # Lets test the first one transition = transition_domains.first() expected_transition_domain = TransitionDomain( - id=24, username='alexandra.bobbitt5@test.com', domain_name='fakewebsite2.gov', status='on hold', @@ -193,10 +192,12 @@ class TestOrganizationMigration(TestCase): state_territory='Oh', zipcode='43268' ) + expected_transition_domain.id = transition.id self.assertEqual(transition, expected_transition_domain) def test_load_organization_data_domain_information(self): + self.maxDiff = None # == First, parse all existing data == # self.run_load_domains() self.run_transfer_domains() @@ -234,7 +235,7 @@ class TestOrganizationMigration(TestCase): anything_else=None, is_policy_acknowledged=None ) - self.assertEqual(domain_information, expected_domain_information) + self.assertEqual(domain_information.__dict__, expected_domain_information.__dict__) def test_load_organization_data_integrity(self): """Validates data integrity with the load_org_data command""" From 2f9896686bd304998c5358ff2c5f400a56537fa1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:20:31 -0700 Subject: [PATCH 027/119] Update test_transition_domain_migrations.py --- .../test_transition_domain_migrations.py | 33 +++---------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 9ae9a66be..3e75c6b4e 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -208,34 +208,11 @@ class TestOrganizationMigration(TestCase): # == Third, test that we've loaded data as we expect == # _domain = Domain.objects.filter(name="fakewebsite2.gov").get() domain_information = DomainInformation.objects.filter(domain=_domain).get() - expected_domain_information = DomainInformation( - id=4, - creator_id=1, - domain_application_id=None, - organization_type='federal', - federally_recognized_tribe=None, - state_recognized_tribe=None, - tribe_name=None, - federal_agency='Department of Commerce', - federal_type='executive', - is_election_board=None, - organization_name='Fanoodle', - address_line1='93001 Arizona Drive', - address_line2=None, - city='Columbus', - state_territory='Oh', - zipcode='43268', - urbanization=None, - about_your_organization=None, - authorizing_official_id=5, - domain_id=4, - submitter_id=None, - purpose=None, - no_other_contacts_rationale=None, - anything_else=None, - is_policy_acknowledged=None - ) - self.assertEqual(domain_information.__dict__, expected_domain_information.__dict__) + + self.assertEqual(domain_information.address_line1, '93001 Arizona Drive') + self.assertEqual(domain_information.city, 'Columbus') + self.assertEqual(domain_information.state_territory, 'Oh') + self.assertEqual(domain_information.zipcode, '43268') def test_load_organization_data_integrity(self): """Validates data integrity with the load_org_data command""" From b875a4583d0e15edad0b8330b32ed6ff64bde79c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 15:14:02 -0700 Subject: [PATCH 028/119] Simplify Load_Organization_data --- .../commands/load_organization_data.py | 175 +++++++----------- .../utility/extra_transition_domain_helper.py | 56 +++--- .../utility/transition_domain_arguments.py | 2 +- .../test_transition_domain_migrations.py | 59 +++--- 4 files changed, 120 insertions(+), 172 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 95743c6b8..ae9f7a29b 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -105,10 +105,7 @@ class Command(BaseCommand): if not proceed: return None - logger.info( - f"{TerminalColors.MAGENTA}" - "Loading organization data onto TransitionDomain tables..." - ) + logger.info(f"{TerminalColors.MAGENTA}" "Loading organization data onto TransitionDomain tables...") load = OrganizationDataLoader(args) domain_information_to_update = load.update_organization_data_for_all() @@ -135,11 +132,7 @@ class Command(BaseCommand): return None if len(domain_information_to_update) == 0: - logger.error( - f"{TerminalColors.MAGENTA}" - "No DomainInformation objects exist" - f"{TerminalColors.ENDC}" - ) + logger.error(f"{TerminalColors.MAGENTA}" "No DomainInformation objects exist" f"{TerminalColors.ENDC}") return None logger.info( @@ -148,125 +141,93 @@ class Command(BaseCommand): f"{TerminalColors.ENDC}" ) self.update_domain_information(domain_information_to_update, args.debug) - + def update_domain_information(self, desired_objects: List[TransitionDomain], debug): di_to_update = [] di_failed_to_update = [] - # These are fields that we COULD update, but fields we choose not to update. - # For instance, if the user already entered data - lets not corrupt that. di_skipped = [] - # Grab each TransitionDomain we want to change. Store it. - # Fetches all TransitionDomains in one query. + # Grab each TransitionDomain we want to change. transition_domains = TransitionDomain.objects.filter( username__in=[item.username for item in desired_objects], - domain_name__in=[item.domain_name for item in desired_objects] + domain_name__in=[item.domain_name for item in desired_objects], ).distinct() if len(desired_objects) != len(transition_domains): raise Exception("Could not find all desired TransitionDomains") # Then, for each domain_name grab the associated domain object. - # Fetches all Domains in one query. - domains = Domain.objects.filter( - name__in=[td.domain_name for td in transition_domains] - ) - + domains = Domain.objects.filter(name__in=[td.domain_name for td in transition_domains]) + # Create dictionary for faster lookup + domains_dict = {d.name: d for d in domains} # Start with all DomainInformation objects filtered_domain_informations = DomainInformation.objects.all() - - changed_fields = [ - "address_line1", - "city", - "state_territory", - "zipcode", - ] - - # Chain filter calls for each field. This checks to see if the end user - # made a change to ANY field in changed_fields. If they did, don't update their information. - # We assume that if they made a change, we don't want to interfere with that. - for field in changed_fields: - # For each changed_field, check if no data exists - filtered_domain_informations = filtered_domain_informations.filter(**{f'{field}__isnull': True}) # Then, use each domain object to map domain <--> DomainInformation # Fetches all DomainInformations in one query. + # If any related organization fields have been updated, + # we can assume that they modified this information themselves - thus we should not update it. domain_informations = filtered_domain_informations.filter( - domain__in=domains + domain__in=domains, + address_line1__isnull=True, + city__isnull=True, + state_territory__isnull=True, + zipcode__isnull=True, ) - # Create dictionaries for faster lookup - domains_dict = {d.name: d for d in domains} domain_informations_dict = {di.domain.name: di for di in domain_informations} for item in transition_domains: - try: - should_update = True - # Grab the current Domain. This ensures we are pointing towards the right place. - if item.domain_name not in domains_dict: - logger.error(f"Could not add {item.domain_name}. Domain does not exist.") - di_failed_to_update.append(item) - continue - current_domain = domains_dict[item.domain_name] - - # Based on the current domain, grab the right DomainInformation object. - if current_domain.name in domain_informations_dict: - current_domain_information = domain_informations_dict[current_domain.name] - current_domain_information.address_line1 = item.address_line - current_domain_information.city = item.city - current_domain_information.state_territory = item.state_territory - current_domain_information.zipcode = item.zipcode - - if debug: - logger.info(f"Updating {current_domain.name}...") - - else: - logger.info( - f"{TerminalColors.YELLOW}" - f"Domain {current_domain.name} was updated by a user. Cannot update." - f"{TerminalColors.ENDC}" - ) - should_update = False - - except Exception as err: - logger.error(err) + if item.domain_name not in domains_dict: + logger.error(f"Could not add {item.domain_name}. Domain does not exist.") di_failed_to_update.append(item) - else: - if should_update: - di_to_update.append(current_domain_information) - else: - # TODO either update to name for all, - # or have this filter to the right field - di_skipped.append(item) - - if len(di_failed_to_update) > 0: + continue + + current_domain = domains_dict[item.domain_name] + if current_domain.name not in domain_informations_dict: + logger.info( + f"{TerminalColors.YELLOW}" + f"Domain {current_domain.name} was updated by a user. Cannot update." + f"{TerminalColors.ENDC}" + ) + di_skipped.append(item) + continue + + # Based on the current domain, grab the right DomainInformation object. + current_domain_information = domain_informations_dict[current_domain.name] + + # Update fields + current_domain_information.address_line1 = item.address_line + current_domain_information.city = item.city + current_domain_information.state_territory = item.state_territory + current_domain_information.zipcode = item.zipcode + + di_to_update.append(current_domain_information) + if debug: + logger.info(f"Updated {current_domain.name}...") + + if di_failed_to_update: + failed = [item.domain_name for item in di_failed_to_update] logger.error( - f"{TerminalColors.FAIL}" - "Failed to update. An exception was encountered " - f"on the following TransitionDomains: {[item for item in di_failed_to_update]}" - f"{TerminalColors.ENDC}" + f"""{TerminalColors.FAIL} + Failed to update. An exception was encountered on the following TransitionDomains: {failed} + {TerminalColors.ENDC}""" ) raise Exception("Failed to update DomainInformations") - - skipped_count = len(di_skipped) - if skipped_count > 0: - logger.info(f"Skipped updating {skipped_count} fields. User-supplied data exists") - if not debug: - logger.info( - f"Ready to update {len(di_to_update)} TransitionDomains." - ) - else: - logger.info( - f"Ready to update {len(di_to_update)} TransitionDomains: {[item for item in di_to_update]}" - ) - - logger.info( - f"{TerminalColors.MAGENTA}" - "Beginning mass DomainInformation update..." - f"{TerminalColors.ENDC}" - ) + if di_skipped: + logger.info(f"Skipped updating {len(di_skipped)} fields. User-supplied data exists") + + self.bulk_update_domain_information(di_to_update, debug) + + def bulk_update_domain_information(self, di_to_update, debug): + if debug: + logger.info(f"Updating these TransitionDomains: {[item for item in di_to_update]}") + + logger.info(f"Ready to update {len(di_to_update)} TransitionDomains.") + + logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass DomainInformation update..." f"{TerminalColors.ENDC}") changed_fields = [ "address_line1", @@ -283,15 +244,9 @@ class Command(BaseCommand): page = paginator.page(page_num) DomainInformation.objects.bulk_update(page.object_list, changed_fields) - if not debug: - logger.info( - f"{TerminalColors.OKGREEN}" - f"Updated {len(di_to_update)} DomainInformations." - f"{TerminalColors.ENDC}" - ) - else: - logger.info( - f"{TerminalColors.OKGREEN}" - f"Updated {len(di_to_update)} DomainInformations: {[item for item in di_to_update]}" - f"{TerminalColors.ENDC}" - ) + if debug: + logger.info(f"Updated these DomainInformations: {[item for item in di_to_update]}") + + logger.info( + f"{TerminalColors.OKGREEN}" f"Updated {len(di_to_update)} DomainInformations." f"{TerminalColors.ENDC}" + ) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index be84e7681..5fcab8f82 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -751,26 +751,28 @@ class FileDataHolder: full_filename = date + "." + filename_without_date return (full_filename, can_infer) + class OrganizationDataLoader: """Saves organization data onto Transition Domains. Handles file parsing.""" + def __init__(self, options: TransitionDomainArguments): # Globally stores event logs and organizes them self.parse_logs = FileTransitionLog() self.debug = options.debug options.pattern_map_params = [ - ( - EnumFilenames.DOMAIN_ADDITIONAL, - options.domain_additional_filename, - DomainAdditionalData, - "domainname", - ), - ( - EnumFilenames.ORGANIZATION_ADHOC, - options.organization_adhoc_filename, - OrganizationAdhoc, - "orgid", - ), + ( + EnumFilenames.DOMAIN_ADDITIONAL, + options.domain_additional_filename, + DomainAdditionalData, + "domainname", + ), + ( + EnumFilenames.ORGANIZATION_ADHOC, + options.organization_adhoc_filename, + OrganizationAdhoc, + "orgid", + ), ] # Reads and parses organization data self.parsed_data = ExtraTransitionDomain(options) @@ -779,15 +781,13 @@ class OrganizationDataLoader: self.tds_to_update = [] self.tds_failed_to_update = [] - + def update_organization_data_for_all(self): """Updates org data for all TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() if len(all_transition_domains) < 1: raise Exception( - f"{TerminalColors.FAIL}" - "No TransitionDomains exist. Cannot update." - f"{TerminalColors.ENDC}" + f"{TerminalColors.FAIL}" "No TransitionDomains exist. Cannot update." f"{TerminalColors.ENDC}" ) # Store all actions we want to perform in tds_to_update @@ -822,26 +822,20 @@ class OrganizationDataLoader: if len(self.tds_failed_to_update) > 0: logger.error( - "Failed to update. An exception was encountered " + "Failed to update. An exception was encountered " f"on the following TransitionDomains: {[item for item in self.tds_failed_to_update]}" ) raise Exception("Failed to update TransitionDomains") if not self.debug: - logger.info( - f"Ready to update {len(self.tds_to_update)} TransitionDomains." - ) + logger.info(f"Ready to update {len(self.tds_to_update)} TransitionDomains.") else: logger.info( f"Ready to update {len(self.tds_to_update)} TransitionDomains: {[item for item in self.tds_failed_to_update]}" ) def bulk_update_transition_domains(self, update_list): - logger.info( - f"{TerminalColors.MAGENTA}" - "Beginning mass TransitionDomain update..." - f"{TerminalColors.ENDC}" - ) + logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass TransitionDomain update..." f"{TerminalColors.ENDC}") changed_fields = [ "address_line", @@ -905,7 +899,7 @@ class OrganizationDataLoader: self.log_add_or_changed_values(EnumFilenames.AUTHORITY_ADHOC, changed_fields, domain_name) return transition_domain - + def get_org_info(self, domain_name) -> OrganizationAdhoc: """Maps an id given in get_domain_data to a organization_adhoc record which has its corresponding definition""" @@ -914,17 +908,17 @@ class OrganizationDataLoader: return None org_id = domain_info.orgid return self.get_organization_adhoc(org_id) - + def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: """Grabs a corresponding row within the ORGANIZATION_ADHOC file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) - + def get_domain_data(self, desired_id) -> DomainAdditionalData: """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, based off a desired_id""" return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) - + def get_object_by_id(self, file_type: EnumFilenames, desired_id): """Returns a field in a dictionary based off the type and id. @@ -1032,9 +1026,7 @@ class ExtraTransitionDomain: # metadata about each file and associate it with an enum. # That way if we want the data located at the agency_adhoc file, # we can just call EnumFilenames.AGENCY_ADHOC. - if ( - options.pattern_map_params is None or options.pattern_map_params == [] - ): + if options.pattern_map_params is None or options.pattern_map_params == []: options.pattern_map_params = [ ( EnumFilenames.AGENCY_ADHOC, diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index bfe1dd84e..bd6d8a970 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import List, Optional +from typing import Optional from registrar.management.commands.utility.epp_data_containers import EnumFilenames diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 3e75c6b4e..3b6a04a89 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -18,6 +18,7 @@ from unittest.mock import patch from .common import less_console_noise + class TestOrganizationMigration(TestCase): def setUp(self): """ """ @@ -65,7 +66,7 @@ class TestOrganizationMigration(TestCase): def run_transfer_domains(self): call_command("transfer_transition_domains_to_domains") - + def run_load_organization_data(self): # noqa here (E501) because splitting this up makes it # confusing to read. @@ -162,7 +163,7 @@ class TestOrganizationMigration(TestCase): # == Second, try adding org data to it == # self.run_load_organization_data() - # == Third, test that we've loaded data as we expect == # + # == Third, test that we've loaded data as we expect == # transition_domains = TransitionDomain.objects.filter(domain_name="fakewebsite2.gov") # Should return three objects (three unique emails) @@ -171,33 +172,32 @@ class TestOrganizationMigration(TestCase): # Lets test the first one transition = transition_domains.first() expected_transition_domain = TransitionDomain( - username='alexandra.bobbitt5@test.com', - domain_name='fakewebsite2.gov', - status='on hold', + username="alexandra.bobbitt5@test.com", + domain_name="fakewebsite2.gov", + status="on hold", email_sent=True, - organization_type='Federal', - organization_name='Fanoodle', - federal_type='Executive', - federal_agency='Department of Commerce', + organization_type="Federal", + organization_name="Fanoodle", + federal_type="Executive", + federal_agency="Department of Commerce", epp_creation_date=datetime.date(2004, 5, 7), epp_expiration_date=datetime.date(2023, 9, 30), - first_name='Seline', - middle_name='testmiddle2', - last_name='Tower', + first_name="Seline", + middle_name="testmiddle2", + last_name="Tower", title=None, - email='stower3@answers.com', - phone='151-539-6028', - address_line='93001 Arizona Drive', - city='Columbus', - state_territory='Oh', - zipcode='43268' + email="stower3@answers.com", + phone="151-539-6028", + address_line="93001 Arizona Drive", + city="Columbus", + state_territory="Oh", + zipcode="43268", ) expected_transition_domain.id = transition.id self.assertEqual(transition, expected_transition_domain) - + def test_load_organization_data_domain_information(self): - self.maxDiff = None # == First, parse all existing data == # self.run_load_domains() self.run_transfer_domains() @@ -205,14 +205,14 @@ class TestOrganizationMigration(TestCase): # == Second, try adding org data to it == # self.run_load_organization_data() - # == Third, test that we've loaded data as we expect == # - _domain = Domain.objects.filter(name="fakewebsite2.gov").get() + # == Third, test that we've loaded data as we expect == # + _domain = Domain.objects.filter(name="fakewebsite2.gov").get() domain_information = DomainInformation.objects.filter(domain=_domain).get() - - self.assertEqual(domain_information.address_line1, '93001 Arizona Drive') - self.assertEqual(domain_information.city, 'Columbus') - self.assertEqual(domain_information.state_territory, 'Oh') - self.assertEqual(domain_information.zipcode, '43268') + + self.assertEqual(domain_information.address_line1, "93001 Arizona Drive") + self.assertEqual(domain_information.city, "Columbus") + self.assertEqual(domain_information.state_territory, "Oh") + self.assertEqual(domain_information.zipcode, "43268") def test_load_organization_data_integrity(self): """Validates data integrity with the load_org_data command""" @@ -222,7 +222,7 @@ class TestOrganizationMigration(TestCase): # Second, try adding org data to it self.run_load_organization_data() - + # Third, test that we didn't corrupt any data expected_total_transition_domains = 9 expected_total_domains = 5 @@ -245,6 +245,7 @@ class TestOrganizationMigration(TestCase): expected_missing_domain_invitations, ) + class TestMigrations(TestCase): def setUp(self): """ """ @@ -308,7 +309,7 @@ class TestMigrations(TestCase): migrationJSON=self.migration_json_filename, disablePrompts=True, ) - + def run_load_organization_data(self): # noqa here (E501) because splitting this up makes it # confusing to read. From 70360cd871ddf4a6ab01e53be1344fa9289f7368 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:03:16 -0700 Subject: [PATCH 029/119] Simplified logic --- .../utility/extra_transition_domain_helper.py | 186 +++++++----------- 1 file changed, 74 insertions(+), 112 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 5fcab8f82..b15a93658 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -778,65 +778,45 @@ class OrganizationDataLoader: self.parsed_data = ExtraTransitionDomain(options) # options.infer_filenames will always be false when not SETTING.DEBUG self.parsed_data.parse_all_files(options.infer_filenames) - self.tds_to_update = [] - self.tds_failed_to_update = [] def update_organization_data_for_all(self): """Updates org data for all TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() - if len(all_transition_domains) < 1: + if len(all_transition_domains) == 0: raise Exception( - f"{TerminalColors.FAIL}" "No TransitionDomains exist. Cannot update." f"{TerminalColors.ENDC}" + f"{TerminalColors.FAIL}No TransitionDomains exist. Cannot update.{TerminalColors.ENDC}" ) - # Store all actions we want to perform in tds_to_update self.prepare_transition_domains(all_transition_domains) - # Then if we don't run into any exceptions, bulk_update it + + logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass TransitionDomain update..." f"{TerminalColors.ENDC}") self.bulk_update_transition_domains(self.tds_to_update) + return self.tds_to_update def prepare_transition_domains(self, transition_domains): for item in transition_domains: - try: - updated = self.parse_org_data(item.domain_name, item) - self.tds_to_update.append(updated) - if self.debug: - logger.info(item.display_transition_domain()) - logger.info( - f"Successfully updated TransitionDomain: \n" - f"{TerminalColors.OKCYAN}" - f"{item.display_transition_domain()}" - f"{TerminalColors.ENDC}" - ) - except Exception as err: - logger.error(err) - self.tds_failed_to_update.append(item) - if self.debug: - logger.error( - f"Failed to update TransitionDomain: \n" - f"{TerminalColors.YELLOW}" - f"{item.display_transition_domain()}" - f"{TerminalColors.ENDC}" - ) + updated = self.parse_org_data(item.domain_name, item) + self.tds_to_update.append(updated) + if self.debug: + logger.info( + f"""{TerminalColors.OKCYAN} + Successfully updated: + {item.display_transition_domain()} + {TerminalColors.ENDC}""" + ) - if len(self.tds_failed_to_update) > 0: - logger.error( - "Failed to update. An exception was encountered " - f"on the following TransitionDomains: {[item for item in self.tds_failed_to_update]}" - ) - raise Exception("Failed to update TransitionDomains") + if self.debug: + logger.info(f"Updating the following: {[item for item in self.tds_to_update]}") - if not self.debug: - logger.info(f"Ready to update {len(self.tds_to_update)} TransitionDomains.") - else: - logger.info( - f"Ready to update {len(self.tds_to_update)} TransitionDomains: {[item for item in self.tds_failed_to_update]}" - ) + logger.info( + f"""{TerminalColors.MAGENTA} + Ready to update {len(self.tds_to_update)} TransitionDomains. + {TerminalColors.ENDC}""" + ) def bulk_update_transition_domains(self, update_list): - logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass TransitionDomain update..." f"{TerminalColors.ENDC}") - changed_fields = [ "address_line", "city", @@ -851,19 +831,15 @@ class OrganizationDataLoader: for page_num in paginator.page_range: page = paginator.page(page_num) TransitionDomain.objects.bulk_update(page.object_list, changed_fields) + + if self.debug: + logger.info(f"Updated the following: {[item for item in self.tds_to_update]}") - if not self.debug: - logger.info( - f"{TerminalColors.OKGREEN}" - f"Updated {len(self.tds_to_update)} TransitionDomains." - f"{TerminalColors.ENDC}" - ) - else: - logger.info( - f"{TerminalColors.OKGREEN}" - f"Updated {len(self.tds_to_update)} TransitionDomains: {[item for item in self.tds_failed_to_update]}" - f"{TerminalColors.ENDC}" - ) + logger.info( + f"{TerminalColors.OKGREEN}" + f"Updated {len(self.tds_to_update)} TransitionDomains." + f"{TerminalColors.ENDC}" + ) def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: """Grabs organization_name from the parsed files and associates it @@ -876,7 +852,7 @@ class OrganizationDataLoader: self.parse_logs.create_log_item( EnumFilenames.ORGANIZATION_ADHOC, LogCode.ERROR, - f"Could not add organization_name on {domain_name}, no data exists.", + f"Could not add organization data on {domain_name}, no data exists.", domain_name, not self.debug, ) @@ -888,38 +864,32 @@ class OrganizationDataLoader: transition_domain.state_territory = org_info.orgstate transition_domain.zipcode = org_info.orgzip - # Log what happened to each field. The first value - # is the field name that was updated, second is the value - changed_fields = [ - ("address_line", transition_domain.address_line), - ("city", transition_domain.city), - ("state_territory", transition_domain.state_territory), - ("zipcode", transition_domain.zipcode), - ] - self.log_add_or_changed_values(EnumFilenames.AUTHORITY_ADHOC, changed_fields, domain_name) + if self.debug: + # Log what happened to each field. The first value + # is the field name that was updated, second is the value + changed_fields = [ + ("address_line", transition_domain.address_line), + ("city", transition_domain.city), + ("state_territory", transition_domain.state_territory), + ("zipcode", transition_domain.zipcode), + ] + self.log_add_or_changed_values(changed_fields, domain_name) return transition_domain def get_org_info(self, domain_name) -> OrganizationAdhoc: """Maps an id given in get_domain_data to a organization_adhoc record which has its corresponding definition""" - domain_info = self.get_domain_data(domain_name) - if domain_info is None: + # Get a row in the domain_additional file. The id is the domain_name. + domain_additional_row = self.retrieve_file_data_by_id(EnumFilenames.DOMAIN_ADDITIONAL, domain_name) + if domain_additional_row is None: return None - org_id = domain_info.orgid - return self.get_organization_adhoc(org_id) - def get_organization_adhoc(self, desired_id) -> OrganizationAdhoc: - """Grabs a corresponding row within the ORGANIZATION_ADHOC file, - based off a desired_id""" - return self.get_object_by_id(EnumFilenames.ORGANIZATION_ADHOC, desired_id) + # Get a row in the organization_adhoc file. The id is the orgid in domain_info. + org_row = self.retrieve_file_data_by_id(EnumFilenames.ORGANIZATION_ADHOC, domain_additional_row.orgid) + return org_row - def get_domain_data(self, desired_id) -> DomainAdditionalData: - """Grabs a corresponding row within the DOMAIN_ADDITIONAL file, - based off a desired_id""" - return self.get_object_by_id(EnumFilenames.DOMAIN_ADDITIONAL, desired_id) - - def get_object_by_id(self, file_type: EnumFilenames, desired_id): + def retrieve_file_data_by_id(self, file_type: EnumFilenames, desired_id): """Returns a field in a dictionary based off the type and id. vars: @@ -948,59 +918,51 @@ class OrganizationDataLoader: So, `AuthorityAdhoc(...)` """ # Grabs a dict associated with the file_type. - # For example, EnumFilenames.DOMAIN_ADDITIONAL. - desired_type = self.parsed_data.file_data.get(file_type) - if desired_type is None: - self.parse_logs.create_log_item( - file_type, - LogCode.ERROR, - f"Type {file_type} does not exist", - ) + # For example, EnumFilenames.DOMAIN_ADDITIONAL would map to + # whatever data exists on the domain_additional file. + desired_file = self.parsed_data.file_data.get(file_type) + if desired_file is None: + logger.error(f"Type {file_type} does not exist") return None - # Grab the value given an Id within that file_type dict. - # For example, "igorville.gov". - obj = desired_type.data.get(desired_id) - if obj is None: - self.parse_logs.create_log_item( - file_type, - LogCode.ERROR, - f"Id {desired_id} does not exist for {file_type.value[0]}", - ) - return obj + # This is essentially a dictionary of rows. + data_in_file = desired_file.data - def log_add_or_changed_values(self, file_type, values_to_check, domain_name): + # Get a row in the given file, based on an id. + # For instance, "igorville.gov" in domain_additional. + row_in_file = data_in_file.get(desired_id) + if row_in_file is None: + logger.error(f"Id {desired_id} does not exist for {file_type.value[0]}") + + return row_in_file + + def log_add_or_changed_values(self, values_to_check, domain_name): + """Iterates through a list of fields, and determines if we should log + if the field was added or if the field was updated. + + A field is "updated" when it is not None or not "". + A field is "created" when it is either of those things. + + + """ for field_name, value in values_to_check: str_exists = value is not None and value.strip() != "" # Logs if we either added to this property, # or modified it. self._add_or_change_message( - file_type, field_name, value, domain_name, str_exists, ) - def _add_or_change_message(self, file_type, var_name, changed_value, domain_name, is_update=False): + def _add_or_change_message(self, field_name, changed_value, domain_name, is_update=False): """Creates a log instance when a property is successfully changed on a given TransitionDomain.""" if not is_update: - self.parse_logs.create_log_item( - file_type, - LogCode.INFO, - f"Added {var_name} as '{changed_value}' on {domain_name}", - domain_name, - not self.debug, - ) + logger.info(f"Added {field_name} as '{changed_value}' on {domain_name}") else: - self.parse_logs.create_log_item( - file_type, - LogCode.WARNING, - f"Updated existing {var_name} to '{changed_value}' on {domain_name}", - domain_name, - not self.debug, - ) + logger.warning(f"Updated existing {field_name} to '{changed_value}' on {domain_name}") class ExtraTransitionDomain: From 0552a77c649b3f1bd933d00bc3dbd7510de89646 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:07:57 -0700 Subject: [PATCH 030/119] Linting --- .../utility/extra_transition_domain_helper.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index b15a93658..15932df04 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -757,7 +757,6 @@ class OrganizationDataLoader: def __init__(self, options: TransitionDomainArguments): # Globally stores event logs and organizes them - self.parse_logs = FileTransitionLog() self.debug = options.debug options.pattern_map_params = [ @@ -784,9 +783,7 @@ class OrganizationDataLoader: """Updates org data for all TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() if len(all_transition_domains) == 0: - raise Exception( - f"{TerminalColors.FAIL}No TransitionDomains exist. Cannot update.{TerminalColors.ENDC}" - ) + raise Exception(f"{TerminalColors.FAIL}No TransitionDomains exist. Cannot update.{TerminalColors.ENDC}") self.prepare_transition_domains(all_transition_domains) @@ -831,14 +828,12 @@ class OrganizationDataLoader: for page_num in paginator.page_range: page = paginator.page(page_num) TransitionDomain.objects.bulk_update(page.object_list, changed_fields) - + if self.debug: logger.info(f"Updated the following: {[item for item in self.tds_to_update]}") logger.info( - f"{TerminalColors.OKGREEN}" - f"Updated {len(self.tds_to_update)} TransitionDomains." - f"{TerminalColors.ENDC}" + f"{TerminalColors.OKGREEN}" f"Updated {len(self.tds_to_update)} TransitionDomains." f"{TerminalColors.ENDC}" ) def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: @@ -849,13 +844,7 @@ class OrganizationDataLoader: org_info = self.get_org_info(domain_name) if org_info is None: - self.parse_logs.create_log_item( - EnumFilenames.ORGANIZATION_ADHOC, - LogCode.ERROR, - f"Could not add organization data on {domain_name}, no data exists.", - domain_name, - not self.debug, - ) + logger.error(f"Could not add organization data on {domain_name}, no data exists.") return transition_domain # Add street info @@ -938,12 +927,12 @@ class OrganizationDataLoader: def log_add_or_changed_values(self, values_to_check, domain_name): """Iterates through a list of fields, and determines if we should log - if the field was added or if the field was updated. - + if the field was added or if the field was updated. + A field is "updated" when it is not None or not "". A field is "created" when it is either of those things. - + """ for field_name, value in values_to_check: str_exists = value is not None and value.strip() != "" From 2659cd816bfdda83e87b03672c0ec25f365fe169 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:10:00 -0700 Subject: [PATCH 031/119] Update extra_transition_domain_helper.py --- .../commands/utility/extra_transition_domain_helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 15932df04..68054f27e 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -870,15 +870,15 @@ class OrganizationDataLoader: """Maps an id given in get_domain_data to a organization_adhoc record which has its corresponding definition""" # Get a row in the domain_additional file. The id is the domain_name. - domain_additional_row = self.retrieve_file_data_by_id(EnumFilenames.DOMAIN_ADDITIONAL, domain_name) + domain_additional_row = self.retrieve_row_by_id(EnumFilenames.DOMAIN_ADDITIONAL, domain_name) if domain_additional_row is None: return None # Get a row in the organization_adhoc file. The id is the orgid in domain_info. - org_row = self.retrieve_file_data_by_id(EnumFilenames.ORGANIZATION_ADHOC, domain_additional_row.orgid) + org_row = self.retrieve_row_by_id(EnumFilenames.ORGANIZATION_ADHOC, domain_additional_row.orgid) return org_row - def retrieve_file_data_by_id(self, file_type: EnumFilenames, desired_id): + def retrieve_row_by_id(self, file_type: EnumFilenames, desired_id): """Returns a field in a dictionary based off the type and id. vars: From e853d4ef165ae573c6d3238335133b782b198918 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 08:58:27 -0700 Subject: [PATCH 032/119] Code cleanup --- .../commands/load_organization_data.py | 69 ++++++++----------- src/registrar/utility/errors.py | 27 ++++++++ 2 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index ae9f7a29b..9a15f646e 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -3,17 +3,18 @@ import argparse import json import logging +import os from django.core.management import BaseCommand from registrar.management.commands.utility.extra_transition_domain_helper import OrganizationDataLoader from registrar.management.commands.utility.terminal_helper import TerminalColors, TerminalHelper from registrar.management.commands.utility.transition_domain_arguments import TransitionDomainArguments -from registrar.models import TransitionDomain -from registrar.models.domain import Domain -from registrar.models.domain_information import DomainInformation +from registrar.models import TransitionDomain, DomainInformation from django.core.paginator import Paginator from typing import List +from registrar.utility.errors import LoadOrganizationError, LoadOrganizationErrorCodes + logger = logging.getLogger(__name__) @@ -48,25 +49,15 @@ class Command(BaseCommand): """Process the objects in TransitionDomain.""" # === Parse JSON file === # - # Desired directory for additional TransitionDomain data - # (In the event they are stored seperately) - directory = options["directory"] - # Add a slash if the last character isn't one - if directory and directory[-1] != "/": - directory += "/" - - json_filepath = directory + migration_json_filename + json_filepath = os.path.join(options["directory"], migration_json_filename) # If a JSON was provided, use its values instead of defaults. with open(json_filepath, "r") as jsonFile: # load JSON object as a dictionary try: data = json.load(jsonFile) - # Create an instance of TransitionDomainArguments - # Iterate over the data from the JSON file - for key, value in data.items(): - if value is not None and value.strip() != "": - options[key] = value + # Iterate over the data from the JSON file. Skip any unused values. + options.update({key: value for key, value in data.items() if value is not None and value.strip() != ""}) except Exception as err: logger.error( f"{TerminalColors.FAIL}" @@ -76,6 +67,7 @@ class Command(BaseCommand): ) raise err # === End parse JSON file === # + args = TransitionDomainArguments(**options) changed_fields = [ @@ -90,7 +82,7 @@ class Command(BaseCommand): ==Master data file== domain_additional_filename: {args.domain_additional_filename} - ==Organization name information== + ==Organization data== organization_adhoc_filename: {args.organization_adhoc_filename} ==Containing directory== @@ -151,51 +143,46 @@ class Command(BaseCommand): transition_domains = TransitionDomain.objects.filter( username__in=[item.username for item in desired_objects], domain_name__in=[item.domain_name for item in desired_objects], - ).distinct() + ) + # This indicates that some form of data corruption happened. if len(desired_objects) != len(transition_domains): - raise Exception("Could not find all desired TransitionDomains") - - # Then, for each domain_name grab the associated domain object. - domains = Domain.objects.filter(name__in=[td.domain_name for td in transition_domains]) - # Create dictionary for faster lookup - domains_dict = {d.name: d for d in domains} + raise LoadOrganizationError(code=LoadOrganizationErrorCodes.TRANSITION_DOMAINS_NOT_FOUND) # Start with all DomainInformation objects - filtered_domain_informations = DomainInformation.objects.all() + domain_informations = DomainInformation.objects.all() + domain_informations_dict = {di.domain.name: di for di in domain_informations} - # Then, use each domain object to map domain <--> DomainInformation + # Then, use each domain object to map TransitionDomain <--> DomainInformation # Fetches all DomainInformations in one query. # If any related organization fields have been updated, # we can assume that they modified this information themselves - thus we should not update it. - domain_informations = filtered_domain_informations.filter( - domain__in=domains, + domain_informations = domain_informations.filter( + domain__name__in=[td.domain_name for td in transition_domains], address_line1__isnull=True, city__isnull=True, state_territory__isnull=True, zipcode__isnull=True, ) - domain_informations_dict = {di.domain.name: di for di in domain_informations} - + filtered_domain_informations_dict = {di.domain.name: di for di in domain_informations} for item in transition_domains: - if item.domain_name not in domains_dict: + if item.domain_name not in domain_informations_dict: logger.error(f"Could not add {item.domain_name}. Domain does not exist.") di_failed_to_update.append(item) continue - current_domain = domains_dict[item.domain_name] - if current_domain.name not in domain_informations_dict: + if item.domain_name not in filtered_domain_informations_dict: logger.info( f"{TerminalColors.YELLOW}" - f"Domain {current_domain.name} was updated by a user. Cannot update." + f"Domain {item.domain_name} was updated by a user. Cannot update." f"{TerminalColors.ENDC}" ) di_skipped.append(item) continue # Based on the current domain, grab the right DomainInformation object. - current_domain_information = domain_informations_dict[current_domain.name] + current_domain_information = filtered_domain_informations_dict[item.domain_name] # Update fields current_domain_information.address_line1 = item.address_line @@ -205,27 +192,27 @@ class Command(BaseCommand): di_to_update.append(current_domain_information) if debug: - logger.info(f"Updated {current_domain.name}...") + logger.info(f"Updated {current_domain_information.domain.name}...") if di_failed_to_update: failed = [item.domain_name for item in di_failed_to_update] logger.error( f"""{TerminalColors.FAIL} - Failed to update. An exception was encountered on the following TransitionDomains: {failed} + Failed to update. An exception was encountered on the following DomainInformations: {failed} {TerminalColors.ENDC}""" ) - raise Exception("Failed to update DomainInformations") + raise LoadOrganizationError(code=LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED) if di_skipped: - logger.info(f"Skipped updating {len(di_skipped)} fields. User-supplied data exists") + logger.info(f"Skipped updating {len(di_skipped)} fields. User-supplied data exists.") self.bulk_update_domain_information(di_to_update, debug) def bulk_update_domain_information(self, di_to_update, debug): if debug: - logger.info(f"Updating these TransitionDomains: {[item for item in di_to_update]}") + logger.info(f"Updating these DomainInformations: {[item for item in di_to_update]}") - logger.info(f"Ready to update {len(di_to_update)} TransitionDomains.") + logger.info(f"Ready to update {len(di_to_update)} DomainInformations.") logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass DomainInformation update..." f"{TerminalColors.ENDC}") diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 4ca3a9a12..b9f24eba3 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -57,6 +57,33 @@ contact help@get.gov return f"{self.message}" +class LoadOrganizationErrorCodes(IntEnum): + TRANSITION_DOMAINS_NOT_FOUND = 1 + UPDATE_DOMAIN_INFO_FAILED = 2 + + +class LoadOrganizationError(Exception): + """ + Error class used in the load_organization_data script + """ + + _error_mapping = { + LoadOrganizationErrorCodes.TRANSITION_DOMAINS_NOT_FOUND: ( + "Could not find all desired TransitionDomains. " "(Possible data corruption?)" + ), + LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED: "Failed to update DomainInformation", + } + + def __init__(self, *args, code=None, **kwargs): + super().__init__(*args, **kwargs) + self.code = code + if self.code in self._error_mapping: + self.message = self._error_mapping.get(self.code) + + def __str__(self): + return f"{self.message}" + + class NameserverErrorCodes(IntEnum): """Used in the NameserverError class for error mapping. From 48b9666498b99674e4665846ae9e76eefeaf5ad9 Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Mon, 20 Nov 2023 09:02:35 -0800 Subject: [PATCH 033/119] Save changes --- src/registrar/forms/domain.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 6cb5b338f..e4723d0ec 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -125,6 +125,19 @@ class ContactForm(forms.ModelForm): class Meta: model = Contact fields = ["first_name", "middle_name", "last_name", "title", "email", "phone"] + error_messages = { + "first_name": {"required": "Enter your first name / given name."}, + "last_name": {"required": "Enter your last name / family name."}, + "title": { + "required": "Enter your title or role in your organization (e.g., Chief Information Officer)." + }, + "email": { + "required": "Enter your email address in the required format, like name@example.com." + }, + "phone": { + "required": "Enter your phone number." + }, + } widgets = { "first_name": forms.TextInput, "middle_name": forms.TextInput, From 0ce4a04b4bf78f957e3f96e882bf186409e52616 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:14:10 -0700 Subject: [PATCH 034/119] Add documentation --- docs/operations/data_migration.md | 52 +++++++++++++++++++ .../commands/load_organization_data.py | 14 ++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 7290349ad..fe3d5f45e 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -421,3 +421,55 @@ purposes Used by the migration scripts to trigger a prompt for deleting all table entries. Useful for testing purposes, but *use with caution* + +## Import organization data +During MVP, our import scripts did not populate the following fields: `address_line, city, state_territory, and zipcode`. This was primarily due to time constraints. Because of this, we need to run a follow-on script to load this remaining data on each `DomainInformation` object. + +This script is intended to run under the assumption that the [load_transition_domain](#step-1-load-transition-domains) and the [transfer_transition_domains_to_domains](#step-2-transfer-transition-domain-data-into-main-domain-tables) scripts have already been ran. + +##### LOCAL COMMAND +to run this command locally, enter the following: +```shell +docker compose run -T app ./manage.py load_organization_data {filename_of_migration_json} --debug +``` +* filename_of_migration_filepath_json - This is a [JSON containing a list of filenames](#step-2-obtain-json-file-for-file-locations). This same file was used in the preceeding steps, `load_transition_domain` and `transfer_transition_domains_to_domains`, however, this script only needs two fields: +``` +{ + "domain_additional_filename": "example.domainadditionaldatalink.adhoc.dotgov.txt", + "organization_adhoc_filename": "example.organization.adhoc.dotgov.txt" +} +``` +If you already possess the old JSON, you do not need to modify it. This script can run even if you specify multiple filepaths. It will just skip over unused ones. + +**Example** +```shell +docker compose run -T app ./manage.py load_organization_data migrationFilepaths.json --debug +``` + +##### SANDBOX COMMAND +```shell +./manage.py load_organization_data {filename_of_migration_json} --debug +``` +* **filename_of_migration_filepath_json** - This is a [JSON containing a list of filenames](#step-2-obtain-json-file-for-file-locations). This same file was used in the preceeding steps, `load_transition_domain` and `transfer_transition_domains_to_domains`, however, this script only needs two fields: +``` +{ + "domain_additional_filename": "example.domainadditionaldatalink.adhoc.dotgov.txt", + "organization_adhoc_filename": "example.organization.adhoc.dotgov.txt" +} +``` +If you already possess the old JSON, you do not need to modify it. This script can run even if you specify multiple filepaths. It will just skip over unused ones. + +**Example** +```shell +./manage.py load_organization_data migrationFilepaths.json --debug +``` + +##### Optional parameters +The `load_organization_data` script has five optional parameters. These are as follows: +| | Parameter | Description | +|:-:|:---------------------------------|:----------------------------------------------------------------------------| +| 1 | **sep** | Determines the file separator. Defaults to "\|" | +| 2 | **debug** | Increases logging detail. Defaults to False | +| 3 | **directory** | Specifies the containing directory of the data. Defaults to "migrationdata" | +| 4 | **domain_additional_filename** | Specifies the filename of domain_additional. Used as an override for the JSON. Has no default. | +| 5 | **organization_adhoc_filename** | Specifies the filename of organization_adhoc. Used as an override for the JSON. Has no default. | diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 9a15f646e..3cfba0ca8 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -35,11 +35,13 @@ class Command(BaseCommand): parser.add_argument("--directory", default="migrationdata", help="Desired directory") + # Serves as a domain_additional_filename override parser.add_argument( "--domain_additional_filename", help="Defines the filename for additional domain data", ) + # Serves as a organization_adhoc_filename override parser.add_argument( "--organization_adhoc_filename", help="Defines the filename for domain type adhocs", @@ -56,8 +58,18 @@ class Command(BaseCommand): # load JSON object as a dictionary try: data = json.load(jsonFile) + + skipped_fields = ["domain_additional_filename", "organization_adhoc_filename"] # Iterate over the data from the JSON file. Skip any unused values. - options.update({key: value for key, value in data.items() if value is not None and value.strip() != ""}) + for key, value in data.items(): + if value is not None or value.strip() != "": + continue + + # If any key in skipped_fields has a value, then + # we override what is specified in the JSON. + if key not in skipped_fields: + options[key] = value + except Exception as err: logger.error( f"{TerminalColors.FAIL}" From 04c7bdd3b6b0a5720e851084d0f2511b97e1bf56 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:38:53 -0700 Subject: [PATCH 035/119] Code cleanup --- .../utility/extra_transition_domain_helper.py | 26 +++++-------------- src/registrar/utility/errors.py | 2 ++ 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 68054f27e..0ad6fa2ab 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -12,6 +12,7 @@ import sys from typing import Dict from django.core.paginator import Paginator from registrar.models.transition_domain import TransitionDomain +from registrar.utility.errors import LoadOrganizationError, LoadOrganizationErrorCodes from .epp_data_containers import ( AgencyAdhoc, @@ -756,9 +757,9 @@ class OrganizationDataLoader: """Saves organization data onto Transition Domains. Handles file parsing.""" def __init__(self, options: TransitionDomainArguments): - # Globally stores event logs and organizes them self.debug = options.debug + # We want to data from the domain_additional file and the organization_adhoc file options.pattern_map_params = [ ( EnumFilenames.DOMAIN_ADDITIONAL, @@ -773,17 +774,20 @@ class OrganizationDataLoader: "orgid", ), ] + # Reads and parses organization data self.parsed_data = ExtraTransitionDomain(options) + # options.infer_filenames will always be false when not SETTING.DEBUG self.parsed_data.parse_all_files(options.infer_filenames) + self.tds_to_update = [] def update_organization_data_for_all(self): """Updates org data for all TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() if len(all_transition_domains) == 0: - raise Exception(f"{TerminalColors.FAIL}No TransitionDomains exist. Cannot update.{TerminalColors.ENDC}") + raise LoadOrganizationError(code=LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE) self.prepare_transition_domains(all_transition_domains) @@ -887,24 +891,6 @@ class OrganizationDataLoader: desired_id: str -> Which id you want to search on. An example would be `"12"` or `"igorville.gov"` - - Explanation: - Each data file has an associated type (file_type) for tracking purposes. - - Each file_type is a dictionary which - contains a dictionary of row[id_field]: object. - - In practice, this would look like: - - EnumFilenames.AUTHORITY_ADHOC: { - "1": AuthorityAdhoc(...), - "2": AuthorityAdhoc(...), - ... - } - - desired_id will then specify which id to grab. If we wanted "1", - then this function will return the value of id "1". - So, `AuthorityAdhoc(...)` """ # Grabs a dict associated with the file_type. # For example, EnumFilenames.DOMAIN_ADDITIONAL would map to diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index b9f24eba3..71ffdb7ed 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -60,6 +60,7 @@ contact help@get.gov class LoadOrganizationErrorCodes(IntEnum): TRANSITION_DOMAINS_NOT_FOUND = 1 UPDATE_DOMAIN_INFO_FAILED = 2 + EMPTY_TRANSITION_DOMAIN_TABLE = 3 class LoadOrganizationError(Exception): @@ -72,6 +73,7 @@ class LoadOrganizationError(Exception): "Could not find all desired TransitionDomains. " "(Possible data corruption?)" ), LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED: "Failed to update DomainInformation", + LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE: "No TransitionDomains exist. Cannot update." } def __init__(self, *args, code=None, **kwargs): From f97dba7ca4c71278d2f4694c43206a4a250996cd Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:50:09 -0700 Subject: [PATCH 036/119] Fix typo --- src/registrar/management/commands/load_organization_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 3cfba0ca8..c9203521d 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -62,12 +62,12 @@ class Command(BaseCommand): skipped_fields = ["domain_additional_filename", "organization_adhoc_filename"] # Iterate over the data from the JSON file. Skip any unused values. for key, value in data.items(): - if value is not None or value.strip() != "": + if value is None or value.strip() == "": continue # If any key in skipped_fields has a value, then # we override what is specified in the JSON. - if key not in skipped_fields: + if key not in skipped_fields or : options[key] = value except Exception as err: From 46dfd7927554cddd6197f56388cdc7dd83b55978 Mon Sep 17 00:00:00 2001 From: Alysia Broddrick Date: Mon, 20 Nov 2023 09:59:22 -0800 Subject: [PATCH 037/119] Add infrastructure for Development environment --- .github/workflows/deploy-development.yaml | 41 +++++++++++++++++++++++ .github/workflows/migrate.yaml | 1 + .github/workflows/reset-db.yaml | 2 +- ops/manifests/manifest-development.yaml | 32 ++++++++++++++++++ ops/scripts/create_dev_sandbox.sh | 6 ++-- src/registrar/config/settings.py | 1 + 6 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/deploy-development.yaml create mode 100644 ops/manifests/manifest-development.yaml diff --git a/.github/workflows/deploy-development.yaml b/.github/workflows/deploy-development.yaml new file mode 100644 index 000000000..13aeb3993 --- /dev/null +++ b/.github/workflows/deploy-development.yaml @@ -0,0 +1,41 @@ +# This workflow runs on pushes to main +# any merge/push to main will result in development being deployed + +name: Build and deploy development for release + +on: + push: + paths-ignore: + - 'docs/**' + - '**.md' + - '.gitignore' + + branches: + - main + +jobs: + deploy-development: + if: ${{ github.ref_type == 'tag' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Compile USWDS assets + working-directory: ./src + run: | + docker compose run node npm install && + docker compose run node npx gulp copyAssets && + docker compose run node npx gulp compile + - name: Collect static assets + working-directory: ./src + run: docker compose run app python manage.py collectstatic --no-input + - name: Deploy to cloud.gov sandbox + uses: 18f/cg-deploy-action@main + env: + DEPLOY_NOW: thanks + with: + cf_username: ${{ secrets.CF_DEVELOPMENT_USERNAME }} + cf_password: ${{ secrets.CF_DEVELOPMENT_PASSWORD }} + cf_org: cisa-dotgov + cf_space: development + push_arguments: "-f ops/manifests/manifest-development.yaml" diff --git a/.github/workflows/migrate.yaml b/.github/workflows/migrate.yaml index 2423c95ad..b1880c830 100644 --- a/.github/workflows/migrate.yaml +++ b/.github/workflows/migrate.yaml @@ -15,6 +15,7 @@ on: options: - stable - staging + - development - ky - es - nl diff --git a/.github/workflows/reset-db.yaml b/.github/workflows/reset-db.yaml index ffc416604..a28270a22 100644 --- a/.github/workflows/reset-db.yaml +++ b/.github/workflows/reset-db.yaml @@ -14,8 +14,8 @@ on: type: choice description: Which environment should we flush and re-load data for? options: - - stable - staging + - development - ky - es - nl diff --git a/ops/manifests/manifest-development.yaml b/ops/manifests/manifest-development.yaml new file mode 100644 index 000000000..0a1f30ffa --- /dev/null +++ b/ops/manifests/manifest-development.yaml @@ -0,0 +1,32 @@ +--- +applications: +- name: getgov-development + buildpacks: + - python_buildpack + path: ../../src + instances: 1 + memory: 512M + stack: cflinuxfs4 + timeout: 180 + command: ./run.sh + health-check-type: http + health-check-http-endpoint: /health + health-check-invocation-timeout: 40 + env: + # Send stdout and stderr straight to the terminal without buffering + PYTHONUNBUFFERED: yup + # Tell Django where to find its configuration + DJANGO_SETTINGS_MODULE: registrar.config.settings + # Tell Django where it is being hosted + DJANGO_BASE_URL: https://getgov-development.app.cloud.gov + # Tell Django how much stuff to log + DJANGO_LOG_LEVEL: INFO + # default public site location + GETGOV_PUBLIC_SITE_URL: https://beta.get.gov + # Flag to disable/enable features in prod environments + IS_PRODUCTION: False + routes: + - route: getgov-development.app.cloud.gov + services: + - getgov-credentials + - getgov-development-database diff --git a/ops/scripts/create_dev_sandbox.sh b/ops/scripts/create_dev_sandbox.sh index 5eeed9c10..c9c55ae29 100755 --- a/ops/scripts/create_dev_sandbox.sh +++ b/ops/scripts/create_dev_sandbox.sh @@ -43,7 +43,7 @@ cp ops/scripts/manifest-sandbox-template.yaml ops/manifests/manifest-$1.yaml sed -i '' "s/ENVIRONMENT/$1/" "ops/manifests/manifest-$1.yaml" echo "Adding new environment to settings.py..." -sed -i '' '/getgov-staging.app.cloud.gov/ {a\ +sed -i '' '/getgov-development.app.cloud.gov/ {a\ '\"getgov-$1.app.cloud.gov\"', }' src/registrar/config/settings.py @@ -105,11 +105,11 @@ echo echo "Moving on to setup Github automation..." echo "Adding new environment to Github Actions..." -sed -i '' '/ - staging/ {a\ +sed -i '' '/ - development/ {a\ - '"$1"' }' .github/workflows/reset-db.yaml -sed -i '' '/ - staging/ {a\ +sed -i '' '/ - development/ {a\ - '"$1"' }' .github/workflows/migrate.yaml diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 5f16f7e3b..584efd459 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -619,6 +619,7 @@ SECURE_SSL_REDIRECT = True ALLOWED_HOSTS = [ "getgov-stable.app.cloud.gov", "getgov-staging.app.cloud.gov", + "getgov-development.app.cloud.gov", "getgov-ky.app.cloud.gov", "getgov-es.app.cloud.gov", "getgov-nl.app.cloud.gov", From 8544c2948f216e1c70feb17f6611637b7448ae2d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:03:25 -0700 Subject: [PATCH 038/119] Fix code typo, v2 --- src/registrar/management/commands/load_organization_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index c9203521d..283ef6f0c 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -67,7 +67,7 @@ class Command(BaseCommand): # If any key in skipped_fields has a value, then # we override what is specified in the JSON. - if key not in skipped_fields or : + if key not in skipped_fields: options[key] = value except Exception as err: From 42ef0262545938f9a9e8ba0abe7532057959216d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:19:55 -0700 Subject: [PATCH 039/119] Update load_organization_data.py --- src/registrar/management/commands/load_organization_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 283ef6f0c..57290a41e 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -67,7 +67,7 @@ class Command(BaseCommand): # If any key in skipped_fields has a value, then # we override what is specified in the JSON. - if key not in skipped_fields: + if options not in skipped_fields or options[key] is not None: options[key] = value except Exception as err: From 2e44a9099a695b3fd5c4401384f7b06125ef1092 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:28:07 -0700 Subject: [PATCH 040/119] Bug fix Accidentally left some stray code from a minor refactor --- src/registrar/management/commands/load_organization_data.py | 2 +- src/registrar/utility/errors.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 57290a41e..2d6d7adff 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -67,7 +67,7 @@ class Command(BaseCommand): # If any key in skipped_fields has a value, then # we override what is specified in the JSON. - if options not in skipped_fields or options[key] is not None: + if options not in skipped_fields: options[key] = value except Exception as err: diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 71ffdb7ed..ba9e9aaa6 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -58,6 +58,12 @@ contact help@get.gov class LoadOrganizationErrorCodes(IntEnum): + """Used when running the load_organization_data script + Overview of error codes: + - 1 TRANSITION_DOMAINS_NOT_FOUND + - 2 UPDATE_DOMAIN_INFO_FAILED + - 3 EMPTY_TRANSITION_DOMAIN_TABLE + """ TRANSITION_DOMAINS_NOT_FOUND = 1 UPDATE_DOMAIN_INFO_FAILED = 2 EMPTY_TRANSITION_DOMAIN_TABLE = 3 From 493c03e7d49c4d9dd15fae416f889dde262eafd9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:53:57 -0700 Subject: [PATCH 041/119] Linting --- .../management/commands/load_organization_data.py | 9 ++++++--- .../commands/utility/extra_transition_domain_helper.py | 6 +++--- src/registrar/utility/errors.py | 5 ++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 2d6d7adff..d98365a9b 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -64,7 +64,7 @@ class Command(BaseCommand): for key, value in data.items(): if value is None or value.strip() == "": continue - + # If any key in skipped_fields has a value, then # we override what is specified in the JSON. if options not in skipped_fields: @@ -163,7 +163,7 @@ class Command(BaseCommand): # Start with all DomainInformation objects domain_informations = DomainInformation.objects.all() - domain_informations_dict = {di.domain.name: di for di in domain_informations} + domain_informations_dict = {di.domain.name: di for di in domain_informations if di.domain is not None} # Then, use each domain object to map TransitionDomain <--> DomainInformation # Fetches all DomainInformations in one query. @@ -177,7 +177,7 @@ class Command(BaseCommand): zipcode__isnull=True, ) - filtered_domain_informations_dict = {di.domain.name: di for di in domain_informations} + filtered_domain_informations_dict = {di.domain.name: di for di in domain_informations if di.domain is not None} for item in transition_domains: if item.domain_name not in domain_informations_dict: logger.error(f"Could not add {item.domain_name}. Domain does not exist.") @@ -196,6 +196,9 @@ class Command(BaseCommand): # Based on the current domain, grab the right DomainInformation object. current_domain_information = filtered_domain_informations_dict[item.domain_name] + if current_domain_information.domain is None or current_domain_information.domain.name is None: + raise LoadOrganizationError(code=LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE) + # Update fields current_domain_information.address_line1 = item.address_line current_domain_information.city = item.city diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 0ad6fa2ab..06b210950 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -9,7 +9,7 @@ import logging import os import sys -from typing import Dict +from typing import Dict, List from django.core.paginator import Paginator from registrar.models.transition_domain import TransitionDomain from registrar.utility.errors import LoadOrganizationError, LoadOrganizationErrorCodes @@ -781,7 +781,7 @@ class OrganizationDataLoader: # options.infer_filenames will always be false when not SETTING.DEBUG self.parsed_data.parse_all_files(options.infer_filenames) - self.tds_to_update = [] + self.tds_to_update: List[TransitionDomain] = [] def update_organization_data_for_all(self): """Updates org data for all TransitionDomains""" @@ -870,7 +870,7 @@ class OrganizationDataLoader: return transition_domain - def get_org_info(self, domain_name) -> OrganizationAdhoc: + def get_org_info(self, domain_name) -> OrganizationAdhoc | None: """Maps an id given in get_domain_data to a organization_adhoc record which has its corresponding definition""" # Get a row in the domain_additional file. The id is the domain_name. diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index ba9e9aaa6..4e0745a2d 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -64,9 +64,11 @@ class LoadOrganizationErrorCodes(IntEnum): - 2 UPDATE_DOMAIN_INFO_FAILED - 3 EMPTY_TRANSITION_DOMAIN_TABLE """ + TRANSITION_DOMAINS_NOT_FOUND = 1 UPDATE_DOMAIN_INFO_FAILED = 2 EMPTY_TRANSITION_DOMAIN_TABLE = 3 + DOMAIN_NAME_WAS_NONE = 4 class LoadOrganizationError(Exception): @@ -79,7 +81,8 @@ class LoadOrganizationError(Exception): "Could not find all desired TransitionDomains. " "(Possible data corruption?)" ), LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED: "Failed to update DomainInformation", - LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE: "No TransitionDomains exist. Cannot update." + LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE: "No TransitionDomains exist. Cannot update.", + LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE: "DomainInformation was updated, but domain was None", } def __init__(self, *args, code=None, **kwargs): From 6f3a27a888ec3c02cf44b0280ec9a68ccc788a81 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 12:16:06 -0700 Subject: [PATCH 042/119] Add more detailed logging --- .../management/commands/load_organization_data.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index d98365a9b..69dafdd4d 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -199,6 +199,15 @@ class Command(BaseCommand): if current_domain_information.domain is None or current_domain_information.domain.name is None: raise LoadOrganizationError(code=LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE) + if item.address_line is None and item.city is None and item.state_territory and item.zipcode is None: + logger.info( + f"{TerminalColors.YELLOW}" + f"Domain {item.domain_name} has no Organization Data. Cannot update." + f"{TerminalColors.ENDC}" + ) + di_skipped.append(item) + continue + # Update fields current_domain_information.address_line1 = item.address_line current_domain_information.city = item.city @@ -219,7 +228,7 @@ class Command(BaseCommand): raise LoadOrganizationError(code=LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED) if di_skipped: - logger.info(f"Skipped updating {len(di_skipped)} fields. User-supplied data exists.") + logger.info(f"Skipped updating {len(di_skipped)} fields. User-supplied data exists, or there is no data.") self.bulk_update_domain_information(di_to_update, debug) From cd6809c9f7bfb56703ba5ffae20f83c1b0b197e6 Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Mon, 20 Nov 2023 13:17:09 -0800 Subject: [PATCH 043/119] Add updated error messages for domain mgmt contact info --- src/registrar/forms/domain.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index e4723d0ec..44dac3a9b 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -125,19 +125,6 @@ class ContactForm(forms.ModelForm): class Meta: model = Contact fields = ["first_name", "middle_name", "last_name", "title", "email", "phone"] - error_messages = { - "first_name": {"required": "Enter your first name / given name."}, - "last_name": {"required": "Enter your last name / family name."}, - "title": { - "required": "Enter your title or role in your organization (e.g., Chief Information Officer)." - }, - "email": { - "required": "Enter your email address in the required format, like name@example.com." - }, - "phone": { - "required": "Enter your phone number." - }, - } widgets = { "first_name": forms.TextInput, "middle_name": forms.TextInput, @@ -160,12 +147,23 @@ class ContactForm(forms.ModelForm): for field_name in self.required: self.fields[field_name].required = True + + # Set custom error messages + self.fields["first_name"].error_messages = {'required': 'Enter your first name / given name.'} + self.fields["last_name"].error_messages = {'required': 'Enter your last name / family name.'} + self.fields["title"].error_messages = { + 'required': 'Enter your title or role in your organization (e.g., Chief Information Officer)' + } + self.fields["email"].error_messages = { + 'required': 'Enter your email address in the required format, like name@example.com.' + } + self.fields["phone"].error_messages = {'required': 'Enter your phone number.'} class DomainSecurityEmailForm(forms.Form): """Form for adding or editing a security email to a domain.""" - security_email = forms.EmailField(label="Security email", required=False) + security_email = forms.EmailField(label="Security email (optional)", required=False) class DomainOrgNameAddressForm(forms.ModelForm): From d2e6e083f3fa512332b2b9a095ceda7b66e092a2 Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:06:44 -0800 Subject: [PATCH 044/119] Add custom errors for authorizing official --- src/registrar/forms/__init__.py | 1 + src/registrar/forms/domain.py | 20 ++++++++++++++++++++ src/registrar/views/domain.py | 3 ++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/registrar/forms/__init__.py b/src/registrar/forms/__init__.py index c3aa89fed..914db375c 100644 --- a/src/registrar/forms/__init__.py +++ b/src/registrar/forms/__init__.py @@ -5,6 +5,7 @@ from .domain import ( DomainSecurityEmailForm, DomainOrgNameAddressForm, ContactForm, + AuthorizingOfficialContactForm, DomainDnssecForm, DomainDsdataFormset, DomainDsdataForm, diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 44dac3a9b..717204019 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -147,6 +147,9 @@ class ContactForm(forms.ModelForm): for field_name in self.required: self.fields[field_name].required = True + + # Set custom form label + self.fields["middle_name"].label = "Middle name (optional)" # Set custom error messages self.fields["first_name"].error_messages = {'required': 'Enter your first name / given name.'} @@ -159,6 +162,23 @@ class ContactForm(forms.ModelForm): } self.fields["phone"].error_messages = {'required': 'Enter your phone number.'} +class AuthorizingOfficialContactForm(ContactForm): + """Form for updating authorizing official contacts.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Set custom error messages + self.fields["first_name"].error_messages = {'required': 'Enter the first name / given name of this contact.'} + self.fields["last_name"].error_messages = {'required': 'Enter the last name / family name of this contact.'} + self.fields["title"].error_messages = { + 'required': 'Enter the title or role in your organization of this contact \ + (e.g., Chief Information Officer).' + } + self.fields["email"].error_messages = { + 'required': 'Enter an email address in the required format, like name@example.com.' + } + self.fields["phone"].error_messages = {'required': 'Enter a phone number for this contact.'} + class DomainSecurityEmailForm(forms.Form): """Form for adding or editing a security email to a domain.""" diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 88fad1567..0dad7115e 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -33,6 +33,7 @@ from registrar.models.utility.contact_error import ContactError from ..forms import ( ContactForm, + AuthorizingOfficialContactForm, DomainOrgNameAddressForm, DomainAddUserForm, DomainSecurityEmailForm, @@ -182,7 +183,7 @@ class DomainAuthorizingOfficialView(DomainFormBaseView): model = Domain template_name = "domain_authorizing_official.html" context_object_name = "domain" - form_class = ContactForm + form_class = AuthorizingOfficialContactForm def get_form_kwargs(self, *args, **kwargs): """Add domain_info.authorizing_official instance to make a bound form.""" From e7db6b8842701dc36c405ffc6cf9ea56825ba7e2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:27:44 -0700 Subject: [PATCH 045/119] Rename variable and move if statement --- .../commands/load_organization_data.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 69dafdd4d..ee193f115 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -111,7 +111,7 @@ class Command(BaseCommand): logger.info(f"{TerminalColors.MAGENTA}" "Loading organization data onto TransitionDomain tables...") load = OrganizationDataLoader(args) - domain_information_to_update = load.update_organization_data_for_all() + transition_domains = load.update_organization_data_for_all() # Reprompt the user to reinspect before updating DomainInformation proceed = TerminalHelper.prompt_for_execution( @@ -127,7 +127,7 @@ class Command(BaseCommand): directory: {args.directory} ==Proposed Changes== - Number of DomainInformation objects to change: {len(domain_information_to_update)} + Number of DomainInformation objects to (potentially) change: {len(transition_domains)} """, prompt_title="Do you wish to load organization data for DomainInformation?", ) @@ -135,8 +135,8 @@ class Command(BaseCommand): if not proceed: return None - if len(domain_information_to_update) == 0: - logger.error(f"{TerminalColors.MAGENTA}" "No DomainInformation objects exist" f"{TerminalColors.ENDC}") + if len(transition_domains) == 0: + logger.error(f"{TerminalColors.MAGENTA}" "No TransitionDomain objects exist" f"{TerminalColors.ENDC}") return None logger.info( @@ -144,7 +144,7 @@ class Command(BaseCommand): "Preparing to load organization data onto DomainInformation tables..." f"{TerminalColors.ENDC}" ) - self.update_domain_information(domain_information_to_update, args.debug) + self.update_domain_information(transition_domains, args.debug) def update_domain_information(self, desired_objects: List[TransitionDomain], debug): di_to_update = [] @@ -183,6 +183,15 @@ class Command(BaseCommand): logger.error(f"Could not add {item.domain_name}. Domain does not exist.") di_failed_to_update.append(item) continue + + if item.address_line is None and item.city is None and item.state_territory and item.zipcode is None: + logger.info( + f"{TerminalColors.YELLOW}" + f"Domain {item.domain_name} has no Organization Data. Cannot update." + f"{TerminalColors.ENDC}" + ) + di_skipped.append(item) + continue if item.domain_name not in filtered_domain_informations_dict: logger.info( @@ -199,15 +208,6 @@ class Command(BaseCommand): if current_domain_information.domain is None or current_domain_information.domain.name is None: raise LoadOrganizationError(code=LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE) - if item.address_line is None and item.city is None and item.state_territory and item.zipcode is None: - logger.info( - f"{TerminalColors.YELLOW}" - f"Domain {item.domain_name} has no Organization Data. Cannot update." - f"{TerminalColors.ENDC}" - ) - di_skipped.append(item) - continue - # Update fields current_domain_information.address_line1 = item.address_line current_domain_information.city = item.city From 060ed333440e5d1386eacef360a3abce94d8b0f5 Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:33:42 -0800 Subject: [PATCH 046/119] Remove custom label --- src/registrar/forms/domain.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 717204019..69ffad8c7 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -147,9 +147,6 @@ class ContactForm(forms.ModelForm): for field_name in self.required: self.fields[field_name].required = True - - # Set custom form label - self.fields["middle_name"].label = "Middle name (optional)" # Set custom error messages self.fields["first_name"].error_messages = {'required': 'Enter your first name / given name.'} From 221534e19dd04419f3a377601f62aff955193ab5 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:49:34 -0700 Subject: [PATCH 047/119] Update comment --- .../commands/utility/extra_transition_domain_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 06b210950..781972077 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -878,7 +878,7 @@ class OrganizationDataLoader: if domain_additional_row is None: return None - # Get a row in the organization_adhoc file. The id is the orgid in domain_info. + # Get a row in the organization_adhoc file. The id is the orgid in domain_additional_row. org_row = self.retrieve_row_by_id(EnumFilenames.ORGANIZATION_ADHOC, domain_additional_row.orgid) return org_row From 0969a76610dd3d538abf8163efc56271c4a15a6f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:53:04 -0700 Subject: [PATCH 048/119] Linting --- src/registrar/management/commands/load_organization_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index ee193f115..2cd2e6514 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -183,7 +183,7 @@ class Command(BaseCommand): logger.error(f"Could not add {item.domain_name}. Domain does not exist.") di_failed_to_update.append(item) continue - + if item.address_line is None and item.city is None and item.state_territory and item.zipcode is None: logger.info( f"{TerminalColors.YELLOW}" From e952d6a9cf426f7a24388637d3acdd8dd0f2adb7 Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:04:31 -0800 Subject: [PATCH 049/119] Fix lint errors --- src/registrar/forms/domain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 69ffad8c7..915699fbe 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -147,7 +147,7 @@ class ContactForm(forms.ModelForm): for field_name in self.required: self.fields[field_name].required = True - + # Set custom error messages self.fields["first_name"].error_messages = {'required': 'Enter your first name / given name.'} self.fields["last_name"].error_messages = {'required': 'Enter your last name / family name.'} @@ -159,6 +159,7 @@ class ContactForm(forms.ModelForm): } self.fields["phone"].error_messages = {'required': 'Enter your phone number.'} + class AuthorizingOfficialContactForm(ContactForm): """Form for updating authorizing official contacts.""" def __init__(self, *args, **kwargs): From f78284cd5e1c3803be18b64f4d13fd72793c3c38 Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:07:27 -0800 Subject: [PATCH 050/119] Fix more lint errors --- src/registrar/forms/domain.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 915699fbe..a9b3cfb83 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -149,33 +149,34 @@ class ContactForm(forms.ModelForm): self.fields[field_name].required = True # Set custom error messages - self.fields["first_name"].error_messages = {'required': 'Enter your first name / given name.'} - self.fields["last_name"].error_messages = {'required': 'Enter your last name / family name.'} + self.fields["first_name"].error_messages = {"required": "Enter your first name / given name."} + self.fields["last_name"].error_messages = {"required": "Enter your last name / family name."} self.fields["title"].error_messages = { - 'required': 'Enter your title or role in your organization (e.g., Chief Information Officer)' + "required": "Enter your title or role in your organization (e.g., Chief Information Officer)" } self.fields["email"].error_messages = { - 'required': 'Enter your email address in the required format, like name@example.com.' + "required": "Enter your email address in the required format, like name@example.com." } - self.fields["phone"].error_messages = {'required': 'Enter your phone number.'} + self.fields["phone"].error_messages = {"required": "Enter your phone number."} class AuthorizingOfficialContactForm(ContactForm): """Form for updating authorizing official contacts.""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Set custom error messages - self.fields["first_name"].error_messages = {'required': 'Enter the first name / given name of this contact.'} - self.fields["last_name"].error_messages = {'required': 'Enter the last name / family name of this contact.'} + self.fields["first_name"].error_messages = {"required": "Enter the first name / given name of this contact."} + self.fields["last_name"].error_messages = {"required": "Enter the last name / family name of this contact."} self.fields["title"].error_messages = { - 'required': 'Enter the title or role in your organization of this contact \ - (e.g., Chief Information Officer).' + "required": "Enter the title or role in your organization of this contact \ + (e.g., Chief Information Officer)." } self.fields["email"].error_messages = { - 'required': 'Enter an email address in the required format, like name@example.com.' + "required": "Enter an email address in the required format, like name@example.com." } - self.fields["phone"].error_messages = {'required': 'Enter a phone number for this contact.'} + self.fields["phone"].error_messages = {"required": "Enter a phone number for this contact."} class DomainSecurityEmailForm(forms.Form): From ba2994812febc6af6de524bf53fd1852652efd36 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:19:33 -0700 Subject: [PATCH 051/119] Split JSON parsing into its own function --- .../commands/load_organization_data.py | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 2cd2e6514..71be71b39 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -51,35 +51,7 @@ class Command(BaseCommand): """Process the objects in TransitionDomain.""" # === Parse JSON file === # - json_filepath = os.path.join(options["directory"], migration_json_filename) - - # If a JSON was provided, use its values instead of defaults. - with open(json_filepath, "r") as jsonFile: - # load JSON object as a dictionary - try: - data = json.load(jsonFile) - - skipped_fields = ["domain_additional_filename", "organization_adhoc_filename"] - # Iterate over the data from the JSON file. Skip any unused values. - for key, value in data.items(): - if value is None or value.strip() == "": - continue - - # If any key in skipped_fields has a value, then - # we override what is specified in the JSON. - if options not in skipped_fields: - options[key] = value - - except Exception as err: - logger.error( - f"{TerminalColors.FAIL}" - "There was an error loading " - "the JSON responsible for providing filepaths." - f"{TerminalColors.ENDC}" - ) - raise err - # === End parse JSON file === # - + options = self.load_json_settings(options, migration_json_filename) args = TransitionDomainArguments(**options) changed_fields = [ @@ -146,6 +118,36 @@ class Command(BaseCommand): ) self.update_domain_information(transition_domains, args.debug) + def load_json_settings(self, options, migration_json_filename): + """Parses options from the given JSON file.""" + json_filepath = os.path.join(options["directory"], migration_json_filename) + + # If a JSON was provided, use its values instead of defaults. + with open(json_filepath, "r") as jsonFile: + # load JSON object as a dictionary + try: + data = json.load(jsonFile) + + skipped_fields = ["domain_additional_filename", "organization_adhoc_filename"] + # Iterate over the data from the JSON file. Skip any unused values. + for key, value in data.items(): + if value is not None and value.strip() != "": + # If any key in skipped_fields has a value, then + # we override what is specified in the JSON. + if options not in skipped_fields: + options[key] = value + + except Exception as err: + logger.error( + f"{TerminalColors.FAIL}" + "There was an error loading " + "the JSON responsible for providing filepaths." + f"{TerminalColors.ENDC}" + ) + raise err + + return options + def update_domain_information(self, desired_objects: List[TransitionDomain], debug): di_to_update = [] di_failed_to_update = [] From 6e9fea5d53431cb66d9e839a7f5873d82654e526 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 20 Nov 2023 17:45:47 -0700 Subject: [PATCH 052/119] Fix for google analytics CSP errors --- src/registrar/config/settings.py | 18 ++++++++++++++++-- src/registrar/templates/base.html | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 5f16f7e3b..3f23a47f7 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -297,12 +297,26 @@ SERVER_EMAIL = "root@get.gov" # Content-Security-Policy configuration # this can be restrictive because we have few external scripts -allowed_sources = ("'self'",) +allowed_sources = ( + "'self'" +) +allowed_sources_scripts = [ + "'self'", + "https://www.googletagmanager.com/", + "https://www.google-analytics.com/" +] CSP_DEFAULT_SRC = allowed_sources -# Most things fall back to default-src, but these two do not and should be +# Most things fall back to default-src, but the following do not and should be # explicitly set CSP_FRAME_ANCESTORS = allowed_sources CSP_FORM_ACTION = allowed_sources +CSP_SCRIPT_SRC_ELEM = allowed_sources_scripts +CSP_SCRIPT_SRC = allowed_sources_scripts +CSP_CONNECT_SRC = allowed_sources_scripts +CSP_INCLUDE_NONCE_IN = [ + 'script-src', + 'script-src-elem' + ] # Cross-Origin Resource Sharing (CORS) configuration # Sets clients that allow access control to manage.get.gov diff --git a/src/registrar/templates/base.html b/src/registrar/templates/base.html index af432e5e9..caecee9f3 100644 --- a/src/registrar/templates/base.html +++ b/src/registrar/templates/base.html @@ -6,7 +6,7 @@ {% if IS_PRODUCTION %} - {% endif %} From c6864b4e505b948e3b832a609358884a35d8f26d Mon Sep 17 00:00:00 2001 From: CocoByte Date: Mon, 20 Nov 2023 18:08:28 -0700 Subject: [PATCH 053/119] linted --- src/registrar/config/settings.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index cdf4a0251..ad6b0cd9d 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -297,14 +297,8 @@ SERVER_EMAIL = "root@get.gov" # Content-Security-Policy configuration # this can be restrictive because we have few external scripts -allowed_sources = ( - "'self'" -) -allowed_sources_scripts = [ - "'self'", - "https://www.googletagmanager.com/", - "https://www.google-analytics.com/" -] +allowed_sources = "'self'" +allowed_sources_scripts = ["'self'", "https://www.googletagmanager.com/", "https://www.google-analytics.com/"] CSP_DEFAULT_SRC = allowed_sources # Most things fall back to default-src, but the following do not and should be # explicitly set @@ -313,10 +307,7 @@ CSP_FORM_ACTION = allowed_sources CSP_SCRIPT_SRC_ELEM = allowed_sources_scripts CSP_SCRIPT_SRC = allowed_sources_scripts CSP_CONNECT_SRC = allowed_sources_scripts -CSP_INCLUDE_NONCE_IN = [ - 'script-src', - 'script-src-elem' - ] +CSP_INCLUDE_NONCE_IN = ["script-src", "script-src-elem"] # Cross-Origin Resource Sharing (CORS) configuration # Sets clients that allow access control to manage.get.gov From d5cdc624b2f4e3bde9ebb0abc1815f7969ba5add Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 08:01:28 -0700 Subject: [PATCH 054/119] Bug fix --- .../utility/extra_transition_domain_helper.py | 77 +++++++++---------- .../utility/transition_domain_arguments.py | 3 - 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 781972077..ed90196ca 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -963,45 +963,44 @@ class ExtraTransitionDomain: # metadata about each file and associate it with an enum. # That way if we want the data located at the agency_adhoc file, # we can just call EnumFilenames.AGENCY_ADHOC. - if options.pattern_map_params is None or options.pattern_map_params == []: - options.pattern_map_params = [ - ( - EnumFilenames.AGENCY_ADHOC, - options.agency_adhoc_filename, - AgencyAdhoc, - "agencyid", - ), - ( - EnumFilenames.DOMAIN_ADDITIONAL, - options.domain_additional_filename, - DomainAdditionalData, - "domainname", - ), - ( - EnumFilenames.DOMAIN_ESCROW, - options.domain_escrow_filename, - DomainEscrow, - "domainname", - ), - ( - EnumFilenames.DOMAIN_ADHOC, - options.domain_adhoc_filename, - DomainTypeAdhoc, - "domaintypeid", - ), - ( - EnumFilenames.ORGANIZATION_ADHOC, - options.organization_adhoc_filename, - OrganizationAdhoc, - "orgid", - ), - ( - EnumFilenames.AUTHORITY_ADHOC, - options.authority_adhoc_filename, - AuthorityAdhoc, - "authorityid", - ), - ] + options.pattern_map_params = [ + ( + EnumFilenames.AGENCY_ADHOC, + options.agency_adhoc_filename, + AgencyAdhoc, + "agencyid", + ), + ( + EnumFilenames.DOMAIN_ADDITIONAL, + options.domain_additional_filename, + DomainAdditionalData, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ESCROW, + options.domain_escrow_filename, + DomainEscrow, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ADHOC, + options.domain_adhoc_filename, + DomainTypeAdhoc, + "domaintypeid", + ), + ( + EnumFilenames.ORGANIZATION_ADHOC, + options.organization_adhoc_filename, + OrganizationAdhoc, + "orgid", + ), + ( + EnumFilenames.AUTHORITY_ADHOC, + options.authority_adhoc_filename, + AuthorityAdhoc, + "authorityid", + ), + ] self.file_data = self.populate_file_data(options.pattern_map_params) diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index bd6d8a970..f941d4edb 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -18,9 +18,6 @@ class TransitionDomainArguments: # Maintains an internal kwargs list and sets values # that match the class definition. def __init__(self, **kwargs): - self.pattern_map_params = [] - if "self.pattern_map_params" in kwargs: - self.pattern_map_params = kwargs["pattern_map_params"] self.kwargs = kwargs for k, v in kwargs.items(): if hasattr(self, k): From a737f26c5000ae3d764d5857672023669d54b240 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 21 Nov 2023 10:31:22 -0500 Subject: [PATCH 055/119] updated error messages in security email and nameserver; updated modal in dnssec ds data --- src/registrar/forms/domain.py | 12 ++++++++++-- src/registrar/templates/domain_dsdata.html | 2 +- src/registrar/utility/errors.py | 17 ++++++----------- src/registrar/views/domain.py | 2 +- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index d15248c10..f7b99555c 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -10,6 +10,8 @@ from registrar.utility.errors import ( NameserverErrorCodes as nsErrorCodes, DsDataError, DsDataErrorCodes, + SecurityEmailError, + SecurityEmailErrorCodes, ) from ..models import Contact, DomainInformation, Domain @@ -156,7 +158,13 @@ class ContactForm(forms.ModelForm): class DomainSecurityEmailForm(forms.Form): """Form for adding or editing a security email to a domain.""" - security_email = forms.EmailField(label="Security email", required=False) + security_email = forms.EmailField( + label="Security email", + required=False, + error_messages={ + 'invalid': str(SecurityEmailError(code=SecurityEmailErrorCodes.BAD_DATA)), + } + ) class DomainOrgNameAddressForm(forms.ModelForm): @@ -237,7 +245,7 @@ class DomainDsdataForm(forms.Form): Tests that string matches all hexadecimal values. Raise validation error to display error in form - if invalid caracters entered + if invalid characters entered """ if not re.match(r"^[0-9a-fA-F]+$", value): raise forms.ValidationError(str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_CHARS))) diff --git a/src/registrar/templates/domain_dsdata.html b/src/registrar/templates/domain_dsdata.html index 927628b11..d049675fc 100644 --- a/src/registrar/templates/domain_dsdata.html +++ b/src/registrar/templates/domain_dsdata.html @@ -114,7 +114,7 @@ aria-describedby="Your DNSSEC records will be deleted from the registry." data-force-action > - {% include 'includes/modal.html' with cancel_button_resets_ds_form=True modal_heading="Warning: You are about to delete all DS records on your domain" modal_description="To fully disable DNSSEC: In addition to deleting your DS records here you’ll also need to delete the DS records at your DNS host. To avoid causing your domain to appear offline you should wait to delete your DS records at your DNS host until the Time to Live (TTL) expires. This is often less than 24 hours, but confirm with your provider." modal_button=modal_button|safe %} + {% include 'includes/modal.html' with cancel_button_resets_ds_form=True modal_heading="Warning: You are about to remove all DS records on your domain" modal_description="To fully disable DNSSEC: In addition to removing your DS records here you’ll also need to delete the DS records at your DNS host. To avoid causing your domain to appear offline you should wait to delete your DS records at your DNS host until the Time to Live (TTL) expires. This is often less than 24 hours, but confirm with your provider." modal_button=modal_button|safe %} {% endblock %} {# domain_content #} diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 2f9daa37d..25148e346 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -66,20 +66,18 @@ class NameserverErrorCodes(IntEnum): value but is not a subdomain - 3 INVALID_IP invalid ip address format or invalid version - 4 TOO_MANY_HOSTS more than the max allowed host values - - 5 UNABLE_TO_UPDATE_DOMAIN unable to update the domain - - 6 MISSING_HOST host is missing for a nameserver - - 7 INVALID_HOST host is invalid for a nameserver - - 8 BAD_DATA bad data input for nameserver + - 5 MISSING_HOST host is missing for a nameserver + - 6 INVALID_HOST host is invalid for a nameserver + - 7 BAD_DATA bad data input for nameserver """ MISSING_IP = 1 GLUE_RECORD_NOT_ALLOWED = 2 INVALID_IP = 3 TOO_MANY_HOSTS = 4 - UNABLE_TO_UPDATE_DOMAIN = 5 - MISSING_HOST = 6 - INVALID_HOST = 7 - BAD_DATA = 8 + MISSING_HOST = 5 + INVALID_HOST = 6 + BAD_DATA = 7 class NameserverError(Exception): @@ -93,9 +91,6 @@ class NameserverError(Exception): NameserverErrorCodes.GLUE_RECORD_NOT_ALLOWED: ("Name server address does not match domain name"), NameserverErrorCodes.INVALID_IP: ("{}: Enter an IP address in the required format."), NameserverErrorCodes.TOO_MANY_HOSTS: ("Too many hosts provided, you may not have more than 13 nameservers."), - NameserverErrorCodes.UNABLE_TO_UPDATE_DOMAIN: ( - "Unable to update domain, changes were not applied. Check logs as a Registry Error is the likely cause" - ), NameserverErrorCodes.MISSING_HOST: ("Name server must be provided to enter IP address."), NameserverErrorCodes.INVALID_HOST: ("Enter a name server in the required format, like ns1.example.com"), NameserverErrorCodes.BAD_DATA: ( diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 616cd9e08..d265f82a9 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -308,7 +308,7 @@ class DomainNameserversView(DomainFormBaseView): except NameserverError as Err: # NamserverErrors *should* be caught in form; if reached here, # there was an uncaught error in submission (through EPP) - messages.error(self.request, NameserverError(code=nsErrorCodes.UNABLE_TO_UPDATE_DOMAIN)) + messages.error(self.request, NameserverError(code=nsErrorCodes.BAD_DATA)) logger.error(f"Nameservers error: {Err}") # TODO: registry is not throwing an error when no connection except RegistryError as Err: From c9c9d7078267d7b3a61db74dc90841a4b2e1683d Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 21 Nov 2023 10:39:30 -0500 Subject: [PATCH 056/119] format for linter --- src/registrar/forms/domain.py | 4 ++-- src/registrar/models/domain.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index f7b99555c..282bbc84f 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -162,8 +162,8 @@ class DomainSecurityEmailForm(forms.Form): label="Security email", required=False, error_messages={ - 'invalid': str(SecurityEmailError(code=SecurityEmailErrorCodes.BAD_DATA)), - } + "invalid": str(SecurityEmailError(code=SecurityEmailErrorCodes.BAD_DATA)), + }, ) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index cb3b784e9..3bd2c0349 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -598,7 +598,7 @@ class Domain(TimeStampedModel, DomainHelper): # if unable to update domain raise error and stop if responseCode != ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY: - raise NameserverError(code=nsErrorCodes.UNABLE_TO_UPDATE_DOMAIN) + raise NameserverError(code=nsErrorCodes.BAD_DATA) successTotalNameservers = len(oldNameservers) - deleteCount + addToDomainCount From 8ef2a8ec026bb424315c648d88fb157725789d54 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 10:28:14 -0700 Subject: [PATCH 057/119] Code clarity refactor --- .../commands/load_organization_data.py | 210 ++++++++++-------- .../utility/extra_transition_domain_helper.py | 77 +++---- .../utility/transition_domain_arguments.py | 2 +- .../test_transition_domain_migrations.py | 11 - 4 files changed, 152 insertions(+), 148 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 71be71b39..0b6fe1f19 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -19,7 +19,22 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): - help = "Send domain invitations once to existing customers." + help = "Load organization data on TransitionDomain and DomainInformation objects" + + def __init__(self): + super().__init__() + self.domain_information_to_update: List[DomainInformation] = [] + + # Stores the domain_name for logging purposes + self.domains_failed_to_update: List[str] = [] + self.domains_skipped: List[str] = [] + + self.changed_fields = [ + "address_line1", + "city", + "state_territory", + "zipcode", + ] def add_arguments(self, parser): """Add command line arguments.""" @@ -49,18 +64,12 @@ class Command(BaseCommand): def handle(self, migration_json_filename, **options): """Process the objects in TransitionDomain.""" - - # === Parse JSON file === # + # Parse JSON file options = self.load_json_settings(options, migration_json_filename) args = TransitionDomainArguments(**options) - changed_fields = [ - "address_line", - "city", - "state_territory", - "zipcode", - ] - proceed = TerminalHelper.prompt_for_execution( + # Will sys.exit() when prompt is "n" + TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, info_to_inspect=f""" ==Master data file== @@ -71,22 +80,16 @@ class Command(BaseCommand): ==Containing directory== directory: {args.directory} - - ==Proposed Changes== - For each TransitionDomain, modify the following fields: {changed_fields} """, prompt_title="Do you wish to load organization data for TransitionDomains?", ) - if not proceed: - return None - - logger.info(f"{TerminalColors.MAGENTA}" "Loading organization data onto TransitionDomain tables...") load = OrganizationDataLoader(args) transition_domains = load.update_organization_data_for_all() # Reprompt the user to reinspect before updating DomainInformation - proceed = TerminalHelper.prompt_for_execution( + # Will sys.exit() when prompt is "n" + TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, info_to_inspect=f""" ==Master data file== @@ -100,23 +103,20 @@ class Command(BaseCommand): ==Proposed Changes== Number of DomainInformation objects to (potentially) change: {len(transition_domains)} + For each DomainInformation, modify the following fields: {self.changed_fields} """, prompt_title="Do you wish to load organization data for DomainInformation?", ) - if not proceed: - return None - - if len(transition_domains) == 0: - logger.error(f"{TerminalColors.MAGENTA}" "No TransitionDomain objects exist" f"{TerminalColors.ENDC}") - return None - logger.info( f"{TerminalColors.MAGENTA}" "Preparing to load organization data onto DomainInformation tables..." f"{TerminalColors.ENDC}" ) - self.update_domain_information(transition_domains, args.debug) + self.prepare_update_domain_information(transition_domains, args.debug) + + logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass DomainInformation update..." f"{TerminalColors.ENDC}") + self.bulk_update_domain_information(args.debug) def load_json_settings(self, options, migration_json_filename): """Parses options from the given JSON file.""" @@ -148,19 +148,19 @@ class Command(BaseCommand): return options - def update_domain_information(self, desired_objects: List[TransitionDomain], debug): - di_to_update = [] - di_failed_to_update = [] - di_skipped = [] + def prepare_update_domain_information(self, target_transition_domains: List[TransitionDomain], debug): + """Returns an array of DomainInformation objects with updated organization data.""" + if len(target_transition_domains) == 0: + raise LoadOrganizationError(code=LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE) # Grab each TransitionDomain we want to change. transition_domains = TransitionDomain.objects.filter( - username__in=[item.username for item in desired_objects], - domain_name__in=[item.domain_name for item in desired_objects], + username__in=[item.username for item in target_transition_domains], + domain_name__in=[item.domain_name for item in target_transition_domains], ) # This indicates that some form of data corruption happened. - if len(desired_objects) != len(transition_domains): + if len(target_transition_domains) != len(transition_domains): raise LoadOrganizationError(code=LoadOrganizationErrorCodes.TRANSITION_DOMAINS_NOT_FOUND) # Start with all DomainInformation objects @@ -178,35 +178,84 @@ class Command(BaseCommand): state_territory__isnull=True, zipcode__isnull=True, ) - filtered_domain_informations_dict = {di.domain.name: di for di in domain_informations if di.domain is not None} + + # === Create DomainInformation objects === # for item in transition_domains: - if item.domain_name not in domain_informations_dict: - logger.error(f"Could not add {item.domain_name}. Domain does not exist.") - di_failed_to_update.append(item) - continue + self.map_transition_domain_to_domain_information( + item, domain_informations_dict, filtered_domain_informations_dict, debug + ) - if item.address_line is None and item.city is None and item.state_territory and item.zipcode is None: - logger.info( - f"{TerminalColors.YELLOW}" - f"Domain {item.domain_name} has no Organization Data. Cannot update." - f"{TerminalColors.ENDC}" - ) - di_skipped.append(item) - continue + # === Log results and return data === # + if len(self.domains_failed_to_update) > 0: + failed = [item.domain_name for item in self.domains_failed_to_update] + logger.error( + f"""{TerminalColors.FAIL} + Failed to update. An exception was encountered on the following Domains: {failed} + {TerminalColors.ENDC}""" + ) + raise LoadOrganizationError(code=LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED) - if item.domain_name not in filtered_domain_informations_dict: - logger.info( - f"{TerminalColors.YELLOW}" - f"Domain {item.domain_name} was updated by a user. Cannot update." - f"{TerminalColors.ENDC}" - ) - di_skipped.append(item) - continue + if debug: + logger.info(f"Updating these DomainInformations: {[item for item in self.domain_information_to_update]}") + if len(self.domains_skipped) > 0: + logger.info( + f"Skipped updating {len(self.domains_skipped)} fields. User-supplied data exists, or there is no data." + ) + + logger.info(f"Ready to update {len(self.domain_information_to_update)} DomainInformations.") + + return self.domain_information_to_update + + def bulk_update_domain_information(self, debug): + """Performs a bulk_update operation on a list of DomainInformation objects""" + # Create a Paginator object. Bulk_update on the full dataset + # is too memory intensive for our current app config, so we can chunk this data instead. + batch_size = 1000 + paginator = Paginator(self.domain_information_to_update, batch_size) + for page_num in paginator.page_range: + page = paginator.page(page_num) + DomainInformation.objects.bulk_update(page.object_list, self.changed_fields) + + if debug: + logger.info(f"Updated these DomainInformations: {[item for item in self.domain_information_to_update]}") + + logger.info( + f"{TerminalColors.OKGREEN}" + f"Updated {len(self.domain_information_to_update)} DomainInformations." + f"{TerminalColors.ENDC}" + ) + + def map_transition_domain_to_domain_information( + self, item, domain_informations_dict, filtered_domain_informations_dict, debug + ): + """Attempts to return a DomainInformation object based on values from TransitionDomain. + Any domains which cannot be updated will be stored in an array. + """ + does_not_exist: bool = self.is_domain_name_missing(item, domain_informations_dict) + all_fields_are_none: bool = self.is_organization_data_missing(item) + user_updated_field: bool = self.is_domain_name_missing(item, filtered_domain_informations_dict) + if does_not_exist: + logger.error(f"Could not add {item.domain_name}. Domain does not exist.") + self.domains_failed_to_update.append(item) + elif all_fields_are_none: + logger.info( + f"{TerminalColors.YELLOW}" + f"Domain {item.domain_name} has no Organization Data. Cannot update." + f"{TerminalColors.ENDC}" + ) + self.domains_skipped.append(item) + elif user_updated_field: + logger.info( + f"{TerminalColors.YELLOW}" + f"Domain {item.domain_name} was updated by a user. Cannot update." + f"{TerminalColors.ENDC}" + ) + self.domains_skipped.append(item) + else: # Based on the current domain, grab the right DomainInformation object. current_domain_information = filtered_domain_informations_dict[item.domain_name] - if current_domain_information.domain is None or current_domain_information.domain.name is None: raise LoadOrganizationError(code=LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE) @@ -215,51 +264,16 @@ class Command(BaseCommand): current_domain_information.city = item.city current_domain_information.state_territory = item.state_territory current_domain_information.zipcode = item.zipcode + self.domain_information_to_update.append(current_domain_information) - di_to_update.append(current_domain_information) if debug: logger.info(f"Updated {current_domain_information.domain.name}...") - if di_failed_to_update: - failed = [item.domain_name for item in di_failed_to_update] - logger.error( - f"""{TerminalColors.FAIL} - Failed to update. An exception was encountered on the following DomainInformations: {failed} - {TerminalColors.ENDC}""" - ) - raise LoadOrganizationError(code=LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED) + def is_domain_name_missing(self, item: TransitionDomain, domain_informations_dict): + """Checks if domain_name is in the supplied dictionary""" + return item.domain_name not in domain_informations_dict - if di_skipped: - logger.info(f"Skipped updating {len(di_skipped)} fields. User-supplied data exists, or there is no data.") - - self.bulk_update_domain_information(di_to_update, debug) - - def bulk_update_domain_information(self, di_to_update, debug): - if debug: - logger.info(f"Updating these DomainInformations: {[item for item in di_to_update]}") - - logger.info(f"Ready to update {len(di_to_update)} DomainInformations.") - - logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass DomainInformation update..." f"{TerminalColors.ENDC}") - - changed_fields = [ - "address_line1", - "city", - "state_territory", - "zipcode", - ] - - batch_size = 1000 - # Create a Paginator object. Bulk_update on the full dataset - # is too memory intensive for our current app config, so we can chunk this data instead. - paginator = Paginator(di_to_update, batch_size) - for page_num in paginator.page_range: - page = paginator.page(page_num) - DomainInformation.objects.bulk_update(page.object_list, changed_fields) - - if debug: - logger.info(f"Updated these DomainInformations: {[item for item in di_to_update]}") - - logger.info( - f"{TerminalColors.OKGREEN}" f"Updated {len(di_to_update)} DomainInformations." f"{TerminalColors.ENDC}" - ) + def is_organization_data_missing(self, item: TransitionDomain): + """Checks if all desired Organization fields to update are none""" + fields = [item.address_line, item.city, item.state_territory, item.zipcode] + return all(field is None for field in fields) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index ed90196ca..781972077 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -963,44 +963,45 @@ class ExtraTransitionDomain: # metadata about each file and associate it with an enum. # That way if we want the data located at the agency_adhoc file, # we can just call EnumFilenames.AGENCY_ADHOC. - options.pattern_map_params = [ - ( - EnumFilenames.AGENCY_ADHOC, - options.agency_adhoc_filename, - AgencyAdhoc, - "agencyid", - ), - ( - EnumFilenames.DOMAIN_ADDITIONAL, - options.domain_additional_filename, - DomainAdditionalData, - "domainname", - ), - ( - EnumFilenames.DOMAIN_ESCROW, - options.domain_escrow_filename, - DomainEscrow, - "domainname", - ), - ( - EnumFilenames.DOMAIN_ADHOC, - options.domain_adhoc_filename, - DomainTypeAdhoc, - "domaintypeid", - ), - ( - EnumFilenames.ORGANIZATION_ADHOC, - options.organization_adhoc_filename, - OrganizationAdhoc, - "orgid", - ), - ( - EnumFilenames.AUTHORITY_ADHOC, - options.authority_adhoc_filename, - AuthorityAdhoc, - "authorityid", - ), - ] + if options.pattern_map_params is None or options.pattern_map_params == []: + options.pattern_map_params = [ + ( + EnumFilenames.AGENCY_ADHOC, + options.agency_adhoc_filename, + AgencyAdhoc, + "agencyid", + ), + ( + EnumFilenames.DOMAIN_ADDITIONAL, + options.domain_additional_filename, + DomainAdditionalData, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ESCROW, + options.domain_escrow_filename, + DomainEscrow, + "domainname", + ), + ( + EnumFilenames.DOMAIN_ADHOC, + options.domain_adhoc_filename, + DomainTypeAdhoc, + "domaintypeid", + ), + ( + EnumFilenames.ORGANIZATION_ADHOC, + options.organization_adhoc_filename, + OrganizationAdhoc, + "orgid", + ), + ( + EnumFilenames.AUTHORITY_ADHOC, + options.authority_adhoc_filename, + AuthorityAdhoc, + "authorityid", + ), + ] self.file_data = self.populate_file_data(options.pattern_map_params) diff --git a/src/registrar/management/commands/utility/transition_domain_arguments.py b/src/registrar/management/commands/utility/transition_domain_arguments.py index f941d4edb..1a57d0d2d 100644 --- a/src/registrar/management/commands/utility/transition_domain_arguments.py +++ b/src/registrar/management/commands/utility/transition_domain_arguments.py @@ -18,7 +18,7 @@ class TransitionDomainArguments: # Maintains an internal kwargs list and sets values # that match the class definition. def __init__(self, **kwargs): - self.kwargs = kwargs + self.pattern_map_params = kwargs.get("pattern_map_params", []) for k, v in kwargs.items(): if hasattr(self, k): setattr(self, k, v) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 3b6a04a89..f2107a941 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -27,17 +27,6 @@ class TestOrganizationMigration(TestCase): # self.master_script = "load_transition_domain", self.test_data_file_location = "registrar/tests/data" - self.test_domain_contact_filename = "test_domain_contacts.txt" - self.test_contact_filename = "test_contacts.txt" - self.test_domain_status_filename = "test_domain_statuses.txt" - - # Files for parsing additional TransitionDomain data - self.test_agency_adhoc_filename = "test_agency_adhoc.txt" - self.test_authority_adhoc_filename = "test_authority_adhoc.txt" - self.test_domain_additional = "test_domain_additional.txt" - self.test_domain_types_adhoc = "test_domain_types_adhoc.txt" - self.test_escrow_domains_daily = "test_escrow_domains_daily" - self.test_organization_adhoc = "test_organization_adhoc.txt" self.migration_json_filename = "test_migrationFilepaths.json" def tearDown(self): From 4467d924a0a40f2b4b7842ce498c282570532d7e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 21 Nov 2023 12:35:04 -0500 Subject: [PATCH 058/119] updated logic on sidebar --- src/registrar/templates/domain_sidebar.html | 26 ++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/registrar/templates/domain_sidebar.html b/src/registrar/templates/domain_sidebar.html index 65c5254e9..6be8f655d 100644 --- a/src/registrar/templates/domain_sidebar.html +++ b/src/registrar/templates/domain_sidebar.html @@ -17,6 +17,7 @@ DNS + {% if request.path|startswith:url %}
  • {% url 'domain-dns-nameservers' pk=domain.id as url %} @@ -34,20 +35,23 @@ > DNSSEC - {% if domain.dnssecdata is not None or request.path|startswith:url and request.path|endswith:'dsdata' %} -
      -
    • - {% url 'domain-dns-dnssec-dsdata' pk=domain.id as url %} - - DS Data - -
      • -
    + {% if request.path|startswith:url %} + {% if domain.dnssecdata is not None or request.path|startswith:url and request.path|endswith:'dsdata' %} +
      +
    • + {% url 'domain-dns-dnssec-dsdata' pk=domain.id as url %} + + DS Data + +
      • +
    + {% endif %} {% endif %}
+ {% endif %}
  • From 051515f702792ac1c50058fc7f9b5bd0076c6d82 Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:39:11 -0800 Subject: [PATCH 059/119] Fix lint errors --- src/registrar/forms/domain.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index a9b3cfb83..98ddba6ac 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -167,16 +167,19 @@ class AuthorizingOfficialContactForm(ContactForm): super().__init__(*args, **kwargs) # Set custom error messages - self.fields["first_name"].error_messages = {"required": "Enter the first name / given name of this contact."} - self.fields["last_name"].error_messages = {"required": "Enter the last name / family name of this contact."} + self.fields["first_name"].error_messages = { + "required": "Enter the first name / given name of your authorizing official." + } + self.fields["last_name"].error_messages = { + "required": "Enter the last name / family name of your authorizing official." + } self.fields["title"].error_messages = { - "required": "Enter the title or role in your organization of this contact \ - (e.g., Chief Information Officer)." + "required": "Enter the title or role your authorizing official has in your organization (e.g., Chief Information Officer)." } self.fields["email"].error_messages = { "required": "Enter an email address in the required format, like name@example.com." } - self.fields["phone"].error_messages = {"required": "Enter a phone number for this contact."} + self.fields["phone"].error_messages = {"required": "Enter a phone number for your authorizing official."} class DomainSecurityEmailForm(forms.Form): From 9b4cf5a806286fe36ee278f05690dd3b2e5669cc Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:40:40 -0800 Subject: [PATCH 060/119] Revert leftover change --- src/registrar/forms/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 98ddba6ac..75f54683f 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -185,7 +185,7 @@ class AuthorizingOfficialContactForm(ContactForm): class DomainSecurityEmailForm(forms.Form): """Form for adding or editing a security email to a domain.""" - security_email = forms.EmailField(label="Security email (optional)", required=False) + security_email = forms.EmailField(label="Security email", required=False) class DomainOrgNameAddressForm(forms.ModelForm): From b2f5884a5cd9393293f343bdc9ad6c955ad5ae1e Mon Sep 17 00:00:00 2001 From: Erin <121973038+erinysong@users.noreply.github.com> Date: Tue, 21 Nov 2023 09:44:10 -0800 Subject: [PATCH 061/119] Fix linting errors --- src/registrar/forms/domain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 75f54683f..68179c0c8 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -174,7 +174,8 @@ class AuthorizingOfficialContactForm(ContactForm): "required": "Enter the last name / family name of your authorizing official." } self.fields["title"].error_messages = { - "required": "Enter the title or role your authorizing official has in your organization (e.g., Chief Information Officer)." + "required": "Enter the title or role your authorizing official has in your \ + organization (e.g., Chief Information Officer)." } self.fields["email"].error_messages = { "required": "Enter an email address in the required format, like name@example.com." From bd89eea21f142b817e7bb00a5d8230871c1818d8 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 10:52:59 -0700 Subject: [PATCH 062/119] Minor refactor --- .../commands/load_organization_data.py | 58 +++++++++++-------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 0b6fe1f19..27daf468b 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -12,6 +12,7 @@ from registrar.management.commands.utility.transition_domain_arguments import Tr from registrar.models import TransitionDomain, DomainInformation from django.core.paginator import Paginator from typing import List +from registrar.models.domain import Domain from registrar.utility.errors import LoadOrganizationError, LoadOrganizationErrorCodes @@ -163,15 +164,10 @@ class Command(BaseCommand): if len(target_transition_domains) != len(transition_domains): raise LoadOrganizationError(code=LoadOrganizationErrorCodes.TRANSITION_DOMAINS_NOT_FOUND) - # Start with all DomainInformation objects - domain_informations = DomainInformation.objects.all() - domain_informations_dict = {di.domain.name: di for di in domain_informations if di.domain is not None} - - # Then, use each domain object to map TransitionDomain <--> DomainInformation - # Fetches all DomainInformations in one query. + # Maps TransitionDomain <--> DomainInformation. # If any related organization fields have been updated, # we can assume that they modified this information themselves - thus we should not update it. - domain_informations = domain_informations.filter( + domain_informations = DomainInformation.objects.filter( domain__name__in=[td.domain_name for td in transition_domains], address_line1__isnull=True, city__isnull=True, @@ -182,9 +178,7 @@ class Command(BaseCommand): # === Create DomainInformation objects === # for item in transition_domains: - self.map_transition_domain_to_domain_information( - item, domain_informations_dict, filtered_domain_informations_dict, debug - ) + self.map_transition_domain_to_domain_information(item, filtered_domain_informations_dict, debug) # === Log results and return data === # if len(self.domains_failed_to_update) > 0: @@ -227,18 +221,14 @@ class Command(BaseCommand): f"{TerminalColors.ENDC}" ) - def map_transition_domain_to_domain_information( - self, item, domain_informations_dict, filtered_domain_informations_dict, debug - ): + def map_transition_domain_to_domain_information(self, item, domain_informations_dict, debug): """Attempts to return a DomainInformation object based on values from TransitionDomain. Any domains which cannot be updated will be stored in an array. """ does_not_exist: bool = self.is_domain_name_missing(item, domain_informations_dict) all_fields_are_none: bool = self.is_organization_data_missing(item) - user_updated_field: bool = self.is_domain_name_missing(item, filtered_domain_informations_dict) if does_not_exist: - logger.error(f"Could not add {item.domain_name}. Domain does not exist.") - self.domains_failed_to_update.append(item) + self.handle_if_domain_name_missing(item.domain_name) elif all_fields_are_none: logger.info( f"{TerminalColors.YELLOW}" @@ -246,16 +236,9 @@ class Command(BaseCommand): f"{TerminalColors.ENDC}" ) self.domains_skipped.append(item) - elif user_updated_field: - logger.info( - f"{TerminalColors.YELLOW}" - f"Domain {item.domain_name} was updated by a user. Cannot update." - f"{TerminalColors.ENDC}" - ) - self.domains_skipped.append(item) else: # Based on the current domain, grab the right DomainInformation object. - current_domain_information = filtered_domain_informations_dict[item.domain_name] + current_domain_information = domain_informations_dict[item.domain_name] if current_domain_information.domain is None or current_domain_information.domain.name is None: raise LoadOrganizationError(code=LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE) @@ -277,3 +260,30 @@ class Command(BaseCommand): """Checks if all desired Organization fields to update are none""" fields = [item.address_line, item.city, item.state_territory, item.zipcode] return all(field is None for field in fields) + + def handle_if_domain_name_missing(self, domain_name): + """ + Infers what to log if we can't find a domain_name and updates the relevant lists. + + This function performs the following checks: + 1. If the domain does not exist, it logs an error and adds the domain name to the `domains_failed_to_update` list. + 2. If the domain was updated by a user, it logs an info message and adds the domain name to the `domains_skipped` list. + 3. If there are duplicate domains, it logs an error and adds the domain name to the `domains_failed_to_update` list. + + Args: + domain_name (str): The name of the domain to check. + """ + domains = Domain.objects.filter(name=domain_name) + if domains.count() == 0: + logger.error(f"Could not add {domain_name}. Domain does not exist.") + self.domains_failed_to_update.append(domain_name) + elif domains.count() == 1: + logger.info( + f"{TerminalColors.YELLOW}" + f"Domain {domain_name} was updated by a user. Cannot update." + f"{TerminalColors.ENDC}" + ) + self.domains_skipped.append(domain_name) + else: + logger.error(f"Could not add {domain_name}. Duplicate domains exist.") + self.domains_failed_to_update.append(domain_name) From 79112522beb48212b011874acae1cd69b3458284 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 10:55:18 -0700 Subject: [PATCH 063/119] Update load_organization_data.py --- .../management/commands/load_organization_data.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 27daf468b..3a4bdc664 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -110,13 +110,17 @@ class Command(BaseCommand): ) logger.info( - f"{TerminalColors.MAGENTA}" - "Preparing to load organization data onto DomainInformation tables..." - f"{TerminalColors.ENDC}" + f"""{TerminalColors.MAGENTA} + Preparing to load organization data onto DomainInformation tables... + {TerminalColors.ENDC}""" ) self.prepare_update_domain_information(transition_domains, args.debug) - logger.info(f"{TerminalColors.MAGENTA}" "Beginning mass DomainInformation update..." f"{TerminalColors.ENDC}") + logger.info( + f"""{TerminalColors.MAGENTA} + Beginning mass DomainInformation update... + {TerminalColors.ENDC}""" + ) self.bulk_update_domain_information(args.debug) def load_json_settings(self, options, migration_json_filename): From 0d47a02bfb5e2cd59655a7d54beac288c1e36a7d Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:01:23 -0700 Subject: [PATCH 064/119] Undo domain invitation bug fix Not needed in this PR --- .../management/commands/send_domain_invitations.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/registrar/management/commands/send_domain_invitations.py b/src/registrar/management/commands/send_domain_invitations.py index 0f8ca1c46..603fbce3a 100644 --- a/src/registrar/management/commands/send_domain_invitations.py +++ b/src/registrar/management/commands/send_domain_invitations.py @@ -152,12 +152,6 @@ class Command(BaseCommand): for domain_name in email_data["domains"]: # self.transition_domains is a queryset so we can sub-select # from it and use the objects to mark them as sent - transition_domains = self.transition_domains.filter(username=this_email, domain_name=domain_name) - if len(transition_domains) == 1: - this_transition_domain = transition_domains.get() - this_transition_domain.email_sent = True - this_transition_domain.save() - elif len(transition_domains) > 1: - logger.error(f"Multiple TransitionDomains exist for {this_email}") - else: - logger.error(f"No TransitionDomain exists for {this_email}") + this_transition_domain = self.transition_domains.get(username=this_email, domain_name=domain_name) + this_transition_domain.email_sent = True + this_transition_domain.save() From fa72f31d242ab169018b998215b72fd574392bc9 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 21 Nov 2023 13:24:06 -0500 Subject: [PATCH 065/119] removed maxlength message on ds data digest --- src/registrar/forms/domain.py | 2 -- src/registrar/tests/test_views.py | 24 ------------------------ src/registrar/utility/errors.py | 11 ++++------- 3 files changed, 4 insertions(+), 33 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 282bbc84f..6a9ea2128 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -280,10 +280,8 @@ class DomainDsdataForm(forms.Form): required=True, label="Digest", validators=[validate_hexadecimal], - max_length=64, error_messages={ "required": "Digest is required.", - "max_length": str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_LENGTH)), }, ) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 334fd2824..86397e96b 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1925,30 +1925,6 @@ class TestDomainDNSSEC(TestDomainOverview): result, str(DsDataError(code=DsDataErrorCodes.INVALID_KEYTAG_SIZE)), count=2, status_code=200 ) - def test_ds_data_form_invalid_digest_length(self): - """DS Data form errors with invalid data (digest too long) - - Uses self.app WebTest because we need to interact with forms. - """ - add_data_page = self.app.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_dsdata.id})) - session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] - self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) - # first two nameservers are required, so if we empty one out we should - # get a form error - add_data_page.forms[0]["form-0-key_tag"] = "1234" - add_data_page.forms[0]["form-0-algorithm"] = "3" - add_data_page.forms[0]["form-0-digest_type"] = "1" - add_data_page.forms[0][ - "form-0-digest" - ] = "1234567890123456789012345678901234567890123456789012345678901234567890" - with less_console_noise(): # swallow logged warning message - result = add_data_page.forms[0].submit() - # form submission was a post with an error, response should be a 200 - # error text appears twice, once at the top of the page, once around - # the field. - self.assertContains( - result, str(DsDataError(code=DsDataErrorCodes.INVALID_DIGEST_LENGTH)), count=2, status_code=200 - ) def test_ds_data_form_invalid_digest_chars(self): """DS Data form errors with invalid data (digest contains non hexadecimal chars) diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 25148e346..420c616cb 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -122,17 +122,15 @@ class DsDataErrorCodes(IntEnum): - 1 BAD_DATA bad data input in ds data - 2 INVALID_DIGEST_SHA1 invalid digest for digest type SHA-1 - 3 INVALID_DIGEST_SHA256 invalid digest for digest type SHA-256 - - 4 INVALID_DIGEST_LENGTH invalid digest length exceeds 64 - - 5 INVALID_DIGEST_CHARS invalid chars in digest - - 6 INVALID_KEYTAG_SIZE invalid key tag size > 65535 + - 4 INVALID_DIGEST_CHARS invalid chars in digest + - 5 INVALID_KEYTAG_SIZE invalid key tag size > 65535 """ BAD_DATA = 1 INVALID_DIGEST_SHA1 = 2 INVALID_DIGEST_SHA256 = 3 - INVALID_DIGEST_LENGTH = 4 - INVALID_DIGEST_CHARS = 5 - INVALID_KEYTAG_SIZE = 6 + INVALID_DIGEST_CHARS = 4 + INVALID_KEYTAG_SIZE = 5 class DsDataError(Exception): @@ -147,7 +145,6 @@ class DsDataError(Exception): ), DsDataErrorCodes.INVALID_DIGEST_SHA1: ("SHA-1 digest must be exactly 40 characters."), DsDataErrorCodes.INVALID_DIGEST_SHA256: ("SHA-256 digest must be exactly 64 characters."), - DsDataErrorCodes.INVALID_DIGEST_LENGTH: ("Digest must be at most 64 characters."), DsDataErrorCodes.INVALID_DIGEST_CHARS: ("Digest must contain only alphanumeric characters [0-9,a-f]."), DsDataErrorCodes.INVALID_KEYTAG_SIZE: ("Key tag must be less than 65535"), } From 0279796bb48069d9df203f64af9e2f157cded6f6 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 21 Nov 2023 13:27:52 -0500 Subject: [PATCH 066/119] removed an extraneous blank line --- src/registrar/tests/test_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 86397e96b..936c344f7 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1925,7 +1925,6 @@ class TestDomainDNSSEC(TestDomainOverview): result, str(DsDataError(code=DsDataErrorCodes.INVALID_KEYTAG_SIZE)), count=2, status_code=200 ) - def test_ds_data_form_invalid_digest_chars(self): """DS Data form errors with invalid data (digest contains non hexadecimal chars) From 464791a326526a6cb2ac472337f02183b007f473 Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 21 Nov 2023 12:09:56 -0700 Subject: [PATCH 067/119] Removed CSP_SCRIPT_SRC statement (not actually needed) --- src/registrar/config/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index ad6b0cd9d..240866725 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -305,9 +305,9 @@ CSP_DEFAULT_SRC = allowed_sources CSP_FRAME_ANCESTORS = allowed_sources CSP_FORM_ACTION = allowed_sources CSP_SCRIPT_SRC_ELEM = allowed_sources_scripts -CSP_SCRIPT_SRC = allowed_sources_scripts + CSP_CONNECT_SRC = allowed_sources_scripts -CSP_INCLUDE_NONCE_IN = ["script-src", "script-src-elem"] +CSP_INCLUDE_NONCE_IN = ["script-src-elem"] # Cross-Origin Resource Sharing (CORS) configuration # Sets clients that allow access control to manage.get.gov From 6e32651dad0fa41c0e6e2f35336fffbb4ca4bc9b Mon Sep 17 00:00:00 2001 From: CocoByte Date: Tue, 21 Nov 2023 12:25:59 -0700 Subject: [PATCH 068/119] Further minimized CSP statements --- src/registrar/config/settings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 240866725..60cc90692 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -304,9 +304,8 @@ CSP_DEFAULT_SRC = allowed_sources # explicitly set CSP_FRAME_ANCESTORS = allowed_sources CSP_FORM_ACTION = allowed_sources -CSP_SCRIPT_SRC_ELEM = allowed_sources_scripts - -CSP_CONNECT_SRC = allowed_sources_scripts +CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/"] +CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"] # Cross-Origin Resource Sharing (CORS) configuration From fc8847a36fe9e1e88a002088e7af7de0eb305428 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:09:41 -0700 Subject: [PATCH 069/119] PR suggestions - move errors / test cases --- .../commands/load_organization_data.py | 7 +- .../utility/extra_transition_domain_helper.py | 5 +- .../utility/load_organization_error.py | 39 +++++ .../test_transition_domain_migrations.py | 161 +++++++++++++++--- src/registrar/utility/errors.py | 38 ----- 5 files changed, 184 insertions(+), 66 deletions(-) create mode 100644 src/registrar/management/commands/utility/load_organization_error.py diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 3a4bdc664..4726bcab1 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -14,7 +14,10 @@ from django.core.paginator import Paginator from typing import List from registrar.models.domain import Domain -from registrar.utility.errors import LoadOrganizationError, LoadOrganizationErrorCodes +from registrar.management.commands.utility.load_organization_error import ( + LoadOrganizationError, + LoadOrganizationErrorCodes, +) logger = logging.getLogger(__name__) @@ -276,7 +279,7 @@ class Command(BaseCommand): Args: domain_name (str): The name of the domain to check. - """ + """ # noqa - E501 (harder to read) domains = Domain.objects.filter(name=domain_name) if domains.count() == 0: logger.error(f"Could not add {domain_name}. Domain does not exist.") diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 781972077..75cde995a 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -12,7 +12,10 @@ import sys from typing import Dict, List from django.core.paginator import Paginator from registrar.models.transition_domain import TransitionDomain -from registrar.utility.errors import LoadOrganizationError, LoadOrganizationErrorCodes +from registrar.management.commands.utility.load_organization_error import ( + LoadOrganizationError, + LoadOrganizationErrorCodes, +) from .epp_data_containers import ( AgencyAdhoc, diff --git a/src/registrar/management/commands/utility/load_organization_error.py b/src/registrar/management/commands/utility/load_organization_error.py new file mode 100644 index 000000000..1849558c4 --- /dev/null +++ b/src/registrar/management/commands/utility/load_organization_error.py @@ -0,0 +1,39 @@ +from enum import IntEnum + + +class LoadOrganizationErrorCodes(IntEnum): + """Used when running the load_organization_data script + Overview of error codes: + - 1 TRANSITION_DOMAINS_NOT_FOUND + - 2 UPDATE_DOMAIN_INFO_FAILED + - 3 EMPTY_TRANSITION_DOMAIN_TABLE + """ + + TRANSITION_DOMAINS_NOT_FOUND = 1 + UPDATE_DOMAIN_INFO_FAILED = 2 + EMPTY_TRANSITION_DOMAIN_TABLE = 3 + DOMAIN_NAME_WAS_NONE = 4 + + +class LoadOrganizationError(Exception): + """ + Error class used in the load_organization_data script + """ + + _error_mapping = { + LoadOrganizationErrorCodes.TRANSITION_DOMAINS_NOT_FOUND: ( + "Could not find all desired TransitionDomains. " "(Possible data corruption?)" + ), + LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED: "Failed to update DomainInformation", + LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE: "No TransitionDomains exist. Cannot update.", + LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE: "DomainInformation was updated, but domain was None", + } + + def __init__(self, *args, code=None, **kwargs): + super().__init__(*args, **kwargs) + self.code = code + if self.code in self._error_mapping: + self.message = self._error_mapping.get(self.code) + + def __str__(self): + return f"{self.message}" diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index f2107a941..1f21115ff 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -16,20 +16,19 @@ from registrar.models import ( from django.core.management import call_command from unittest.mock import patch +from registrar.models.contact import Contact + from .common import less_console_noise class TestOrganizationMigration(TestCase): def setUp(self): - """ """ - # self.load_transition_domain_script = "load_transition_domain", - # self.transfer_script = "transfer_transition_domains_to_domains", - # self.master_script = "load_transition_domain", - + """Defines the file name of migration_json and the folder its contained in""" self.test_data_file_location = "registrar/tests/data" self.migration_json_filename = "test_migrationFilepaths.json" def tearDown(self): + """Deletes all DB objects related to migrations""" # Delete domain information Domain.objects.all().delete() DomainInformation.objects.all().delete() @@ -41,6 +40,16 @@ class TestOrganizationMigration(TestCase): UserDomainRole.objects.all().delete() def run_load_domains(self): + """ + This method executes the load_transition_domain command. + + It uses 'unittest.mock.patch' to mock the TerminalHelper.query_yes_no_exit method, + which is a user prompt in the terminal. The mock function always returns True, + allowing the test to proceed without manual user input. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ # noqa here because splitting this up makes it confusing. # ES501 with patch( @@ -54,9 +63,25 @@ class TestOrganizationMigration(TestCase): ) def run_transfer_domains(self): + """ + This method executes the transfer_transition_domains_to_domains command. + + The 'call_command' function from Django's management framework is then used to + execute the load_transition_domain command with the specified arguments. + """ call_command("transfer_transition_domains_to_domains") def run_load_organization_data(self): + """ + This method executes the load_organization_data command. + + It uses 'unittest.mock.patch' to mock the TerminalHelper.query_yes_no_exit method, + which is a user prompt in the terminal. The mock function always returns True, + allowing the test to proceed without manual user input. + + The 'call_command' function from Django's management framework is then used to + execute the load_organization_data command with the specified arguments. + """ # noqa here (E501) because splitting this up makes it # confusing to read. with patch( @@ -88,7 +113,7 @@ class TestOrganizationMigration(TestCase): duplicate_domains = [] missing_domain_informations = [] missing_domain_invites = [] - for transition_domain in TransitionDomain.objects.all(): # DEBUG: + for transition_domain in TransitionDomain.objects.all(): transition_domain_name = transition_domain.domain_name transition_domain_email = transition_domain.username @@ -121,19 +146,6 @@ class TestOrganizationMigration(TestCase): total_domain_informations = len(DomainInformation.objects.all()) total_domain_invitations = len(DomainInvitation.objects.all()) - print( - f""" - total_missing_domains = {len(missing_domains)} - total_duplicate_domains = {len(duplicate_domains)} - total_missing_domain_informations = {len(missing_domain_informations)} - total_missing_domain_invitations = {total_missing_domain_invitations} - - total_transition_domains = {len(TransitionDomain.objects.all())} - total_domains = {len(Domain.objects.all())} - total_domain_informations = {len(DomainInformation.objects.all())} - total_domain_invitations = {len(DomainInvitation.objects.all())} - """ - ) self.assertEqual(total_missing_domains, expected_missing_domains) self.assertEqual(total_duplicate_domains, expected_duplicate_domains) self.assertEqual(total_missing_domain_informations, expected_missing_domain_informations) @@ -145,6 +157,17 @@ class TestOrganizationMigration(TestCase): self.assertEqual(total_domain_invitations, expected_total_domain_invitations) def test_load_organization_data_transition_domain(self): + """ + This test verifies the functionality of the load_organization_data method for TransitionDomain objects. + + The test follows these steps: + 1. Parses all existing data by running the load_domains and transfer_domains methods. + 2. Attempts to add organization data to the parsed data by running the load_organization_data method. + 3. Checks that the data has been loaded as expected. + + The expected result is a set of TransitionDomain objects with specific attributes. + The test fetches the actual TransitionDomain objects from the database and compares them with the expected objects. + """ # noqa - E501 (harder to read) # == First, parse all existing data == # self.run_load_domains() self.run_transfer_domains() @@ -187,6 +210,18 @@ class TestOrganizationMigration(TestCase): self.assertEqual(transition, expected_transition_domain) def test_load_organization_data_domain_information(self): + """ + This test verifies the functionality of the load_organization_data method. + + The test follows these steps: + 1. Parses all existing data by running the load_domains and transfer_domains methods. + 2. Attempts to add organization data to the parsed data by running the load_organization_data method. + 3. Checks that the data has been loaded as expected. + + The expected result is a DomainInformation object with specific attributes. + The test fetches the actual DomainInformation object from the database + and compares it with the expected object. + """ # == First, parse all existing data == # self.run_load_domains() self.run_transfer_domains() @@ -198,13 +233,90 @@ class TestOrganizationMigration(TestCase): _domain = Domain.objects.filter(name="fakewebsite2.gov").get() domain_information = DomainInformation.objects.filter(domain=_domain).get() - self.assertEqual(domain_information.address_line1, "93001 Arizona Drive") - self.assertEqual(domain_information.city, "Columbus") - self.assertEqual(domain_information.state_territory, "Oh") - self.assertEqual(domain_information.zipcode, "43268") + expected_creator = User.objects.filter(username="System").get() + expected_ao = Contact.objects.filter(first_name="Seline", middle_name="testmiddle2", last_name="Tower").get() + expected_domain_information = DomainInformation( + creator=expected_creator, + organization_type="federal", + federal_agency="Department of Commerce", + federal_type="executive", + organization_name="Fanoodle", + address_line1="93001 Arizona Drive", + city="Columbus", + state_territory="Oh", + zipcode="43268", + authorizing_official=expected_ao, + domain=_domain, + ) + # Given that these are different objects, this needs to be set + expected_domain_information.id = domain_information.id + self.assertEqual(domain_information, expected_domain_information) + + def test_load_organization_data_preserves_existing_data(self): + """ + This test verifies that the load_organization_data method does not overwrite existing data. + + The test follows these steps: + 1. Parses all existing data by running the load_domains and transfer_domains methods. + 2. Adds pre-existing fake data to a DomainInformation object and saves it to the database. + 3. Runs the load_organization_data method. + 4. Checks that the pre-existing data in the DomainInformation object has not been overwritten. + + The expected result is that the DomainInformation object retains its pre-existing data + after the load_organization_data method is run. + """ + # == First, parse all existing data == # + self.run_load_domains() + self.run_transfer_domains() + + # == Second, try add prexisting fake data == # + _domain_old = Domain.objects.filter(name="fakewebsite2.gov").get() + domain_information_old = DomainInformation.objects.filter(domain=_domain_old).get() + domain_information_old.address_line1 = "93001 Galactic Way" + domain_information_old.city = "Olympus" + domain_information_old.state_territory = "MA" + domain_information_old.zipcode = "12345" + domain_information_old.save() + + # == Third, try running the script == # + self.run_load_organization_data() + + # == Fourth, test that no data is overwritten as we expect == # + _domain = Domain.objects.filter(name="fakewebsite2.gov").get() + domain_information = DomainInformation.objects.filter(domain=_domain).get() + + expected_creator = User.objects.filter(username="System").get() + expected_ao = Contact.objects.filter(first_name="Seline", middle_name="testmiddle2", last_name="Tower").get() + expected_domain_information = DomainInformation( + creator=expected_creator, + organization_type="federal", + federal_agency="Department of Commerce", + federal_type="executive", + organization_name="Fanoodle", + address_line1="93001 Galactic Way", + city="Olympus", + state_territory="MA", + zipcode="12345", + authorizing_official=expected_ao, + domain=_domain, + ) + # Given that these are different objects, this needs to be set + expected_domain_information.id = domain_information.id + self.assertEqual(domain_information, expected_domain_information) def test_load_organization_data_integrity(self): - """Validates data integrity with the load_org_data command""" + """ + This test verifies the data integrity after running the load_organization_data method. + + The test follows these steps: + 1. Parses all existing data by running the load_domains and transfer_domains methods. + 2. Attempts to add organization data to the parsed data by running the load_organization_data method. + 3. Checks that the data has not been corrupted by comparing the actual counts of objects in the database + with the expected counts. + + The expected result is that the counts of objects in the database + match the expected counts, indicating that the data has not been corrupted. + """ # First, parse all existing data self.run_load_domains() self.run_transfer_domains() @@ -221,7 +333,6 @@ class TestOrganizationMigration(TestCase): expected_missing_domains = 0 expected_duplicate_domains = 0 expected_missing_domain_informations = 0 - # we expect 1 missing invite from anomaly.gov (an injected error) expected_missing_domain_invitations = 1 self.compare_tables( expected_total_transition_domains, diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 4e0745a2d..4ca3a9a12 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -57,44 +57,6 @@ contact help@get.gov return f"{self.message}" -class LoadOrganizationErrorCodes(IntEnum): - """Used when running the load_organization_data script - Overview of error codes: - - 1 TRANSITION_DOMAINS_NOT_FOUND - - 2 UPDATE_DOMAIN_INFO_FAILED - - 3 EMPTY_TRANSITION_DOMAIN_TABLE - """ - - TRANSITION_DOMAINS_NOT_FOUND = 1 - UPDATE_DOMAIN_INFO_FAILED = 2 - EMPTY_TRANSITION_DOMAIN_TABLE = 3 - DOMAIN_NAME_WAS_NONE = 4 - - -class LoadOrganizationError(Exception): - """ - Error class used in the load_organization_data script - """ - - _error_mapping = { - LoadOrganizationErrorCodes.TRANSITION_DOMAINS_NOT_FOUND: ( - "Could not find all desired TransitionDomains. " "(Possible data corruption?)" - ), - LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED: "Failed to update DomainInformation", - LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE: "No TransitionDomains exist. Cannot update.", - LoadOrganizationErrorCodes.DOMAIN_NAME_WAS_NONE: "DomainInformation was updated, but domain was None", - } - - def __init__(self, *args, code=None, **kwargs): - super().__init__(*args, **kwargs) - self.code = code - if self.code in self._error_mapping: - self.message = self._error_mapping.get(self.code) - - def __str__(self): - return f"{self.message}" - - class NameserverErrorCodes(IntEnum): """Used in the NameserverError class for error mapping. From ee2bb38e155c49f2774965ba4ec91bd46935491a Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 21 Nov 2023 13:12:02 -0700 Subject: [PATCH 070/119] Update src/registrar/config/settings.py Co-authored-by: Neil MartinsenBurrell --- src/registrar/config/settings.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 60cc90692..9db15e93f 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -304,6 +304,10 @@ CSP_DEFAULT_SRC = allowed_sources # explicitly set CSP_FRAME_ANCESTORS = allowed_sources CSP_FORM_ACTION = allowed_sources + +# Google analytics requires that we relax our otherwise +# strict CSP by allowing scripts to run from their domain +# and inline with a nonce, as well as allowing connections back to their domain CSP_SCRIPT_SRC_ELEM = ["'self'", "https://www.googletagmanager.com/"] CSP_CONNECT_SRC = ["'self'", "https://www.google-analytics.com/"] CSP_INCLUDE_NONCE_IN = ["script-src-elem"] From 63f67d6f0b6c691431acf5f15b0a446325cf66b7 Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 21 Nov 2023 13:12:14 -0700 Subject: [PATCH 071/119] Update src/registrar/config/settings.py Co-authored-by: Neil MartinsenBurrell --- src/registrar/config/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 9db15e93f..249953f28 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -298,7 +298,6 @@ SERVER_EMAIL = "root@get.gov" # Content-Security-Policy configuration # this can be restrictive because we have few external scripts allowed_sources = "'self'" -allowed_sources_scripts = ["'self'", "https://www.googletagmanager.com/", "https://www.google-analytics.com/"] CSP_DEFAULT_SRC = allowed_sources # Most things fall back to default-src, but the following do not and should be # explicitly set From 9c92c998685f30d31af5714509d81b475f31c755 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:13:20 -0700 Subject: [PATCH 072/119] Removed unused domain invite test --- .../test_transition_domain_migrations.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 1f21115ff..7b04897f3 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -99,11 +99,9 @@ class TestOrganizationMigration(TestCase): expected_total_transition_domains, expected_total_domains, expected_total_domain_informations, - expected_total_domain_invitations, expected_missing_domains, expected_duplicate_domains, expected_missing_domain_informations, - expected_missing_domain_invitations, ): """Does a diff between the transition_domain and the following tables: domain, domain_information and the domain_invitation. @@ -112,20 +110,12 @@ class TestOrganizationMigration(TestCase): missing_domains = [] duplicate_domains = [] missing_domain_informations = [] - missing_domain_invites = [] for transition_domain in TransitionDomain.objects.all(): transition_domain_name = transition_domain.domain_name - transition_domain_email = transition_domain.username - # Check Domain table matching_domains = Domain.objects.filter(name=transition_domain_name) # Check Domain Information table matching_domain_informations = DomainInformation.objects.filter(domain__name=transition_domain_name) - # Check Domain Invitation table - matching_domain_invitations = DomainInvitation.objects.filter( - email=transition_domain_email.lower(), - domain__name=transition_domain_name, - ) if len(matching_domains) == 0: missing_domains.append(transition_domain_name) @@ -133,28 +123,22 @@ class TestOrganizationMigration(TestCase): duplicate_domains.append(transition_domain_name) if len(matching_domain_informations) == 0: missing_domain_informations.append(transition_domain_name) - if len(matching_domain_invitations) == 0: - missing_domain_invites.append(transition_domain_name) total_missing_domains = len(missing_domains) total_duplicate_domains = len(duplicate_domains) total_missing_domain_informations = len(missing_domain_informations) - total_missing_domain_invitations = len(missing_domain_invites) total_transition_domains = len(TransitionDomain.objects.all()) total_domains = len(Domain.objects.all()) total_domain_informations = len(DomainInformation.objects.all()) - total_domain_invitations = len(DomainInvitation.objects.all()) self.assertEqual(total_missing_domains, expected_missing_domains) self.assertEqual(total_duplicate_domains, expected_duplicate_domains) self.assertEqual(total_missing_domain_informations, expected_missing_domain_informations) - self.assertEqual(total_missing_domain_invitations, expected_missing_domain_invitations) self.assertEqual(total_transition_domains, expected_total_transition_domains) self.assertEqual(total_domains, expected_total_domains) self.assertEqual(total_domain_informations, expected_total_domain_informations) - self.assertEqual(total_domain_invitations, expected_total_domain_invitations) def test_load_organization_data_transition_domain(self): """ @@ -328,21 +312,17 @@ class TestOrganizationMigration(TestCase): expected_total_transition_domains = 9 expected_total_domains = 5 expected_total_domain_informations = 5 - expected_total_domain_invitations = 8 expected_missing_domains = 0 expected_duplicate_domains = 0 expected_missing_domain_informations = 0 - expected_missing_domain_invitations = 1 self.compare_tables( expected_total_transition_domains, expected_total_domains, expected_total_domain_informations, - expected_total_domain_invitations, expected_missing_domains, expected_duplicate_domains, expected_missing_domain_informations, - expected_missing_domain_invitations, ) From 38fb3c5eaf0f210ecd15527d7cf106d0b5816532 Mon Sep 17 00:00:00 2001 From: CuriousX Date: Tue, 21 Nov 2023 13:13:31 -0700 Subject: [PATCH 073/119] Update src/registrar/config/settings.py Co-authored-by: Neil MartinsenBurrell --- src/registrar/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index 249953f28..3487b1b66 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -297,7 +297,7 @@ SERVER_EMAIL = "root@get.gov" # Content-Security-Policy configuration # this can be restrictive because we have few external scripts -allowed_sources = "'self'" +allowed_sources = ("'self'",) CSP_DEFAULT_SRC = allowed_sources # Most things fall back to default-src, but the following do not and should be # explicitly set From c6d285d73bb092a86da0bc2cb11fd236bb8138be Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:14:41 -0700 Subject: [PATCH 074/119] Update test_transition_domain_migrations.py --- .../tests/test_transition_domain_migrations.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/registrar/tests/test_transition_domain_migrations.py b/src/registrar/tests/test_transition_domain_migrations.py index 7b04897f3..91625207d 100644 --- a/src/registrar/tests/test_transition_domain_migrations.py +++ b/src/registrar/tests/test_transition_domain_migrations.py @@ -390,19 +390,6 @@ class TestMigrations(TestCase): disablePrompts=True, ) - def run_load_organization_data(self): - # noqa here (E501) because splitting this up makes it - # confusing to read. - with patch( - "registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit", # noqa - return_value=True, - ): - call_command( - "load_organization_data", - self.migration_json_filename, - directory=self.test_data_file_location, - ) - def compare_tables( self, expected_total_transition_domains, From 598d0f643098d108cb2a7ea82b45b4befd150895 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:25:58 -0700 Subject: [PATCH 075/119] Fix logging bug --- .../commands/load_organization_data.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 4726bcab1..87bc62512 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -113,17 +113,13 @@ class Command(BaseCommand): ) logger.info( - f"""{TerminalColors.MAGENTA} - Preparing to load organization data onto DomainInformation tables... - {TerminalColors.ENDC}""" + f"{TerminalColors.MAGENTA}" + "Preparing to load organization data onto DomainInformation tables..." + f"{TerminalColors.ENDC}" ) self.prepare_update_domain_information(transition_domains, args.debug) - logger.info( - f"""{TerminalColors.MAGENTA} - Beginning mass DomainInformation update... - {TerminalColors.ENDC}""" - ) + logger.info(f"{TerminalColors.MAGENTA}" f"Beginning mass DomainInformation update..." f"{TerminalColors.ENDC}") self.bulk_update_domain_information(args.debug) def load_json_settings(self, options, migration_json_filename): @@ -189,10 +185,9 @@ class Command(BaseCommand): # === Log results and return data === # if len(self.domains_failed_to_update) > 0: - failed = [item.domain_name for item in self.domains_failed_to_update] logger.error( f"""{TerminalColors.FAIL} - Failed to update. An exception was encountered on the following Domains: {failed} + Failed to update. An exception was encountered on the following Domains: {self.domains_failed_to_update} {TerminalColors.ENDC}""" ) raise LoadOrganizationError(code=LoadOrganizationErrorCodes.UPDATE_DOMAIN_INFO_FAILED) @@ -201,8 +196,11 @@ class Command(BaseCommand): logger.info(f"Updating these DomainInformations: {[item for item in self.domain_information_to_update]}") if len(self.domains_skipped) > 0: + logger.info(f"Skipped these fields: {self.domains_skipped}") logger.info( + f"{TerminalColors.YELLOW}" f"Skipped updating {len(self.domains_skipped)} fields. User-supplied data exists, or there is no data." + f"{TerminalColors.ENDC}" ) logger.info(f"Ready to update {len(self.domain_information_to_update)} DomainInformations.") @@ -242,7 +240,7 @@ class Command(BaseCommand): f"Domain {item.domain_name} has no Organization Data. Cannot update." f"{TerminalColors.ENDC}" ) - self.domains_skipped.append(item) + self.domains_skipped.append(item.domain_name) else: # Based on the current domain, grab the right DomainInformation object. current_domain_information = domain_informations_dict[item.domain_name] From 0aa54870a8e66971672bf6d28c244768c20316d1 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Tue, 21 Nov 2023 14:46:59 -0700 Subject: [PATCH 076/119] Fix typo and table formatting --- docs/developer/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/developer/README.md b/docs/developer/README.md index 268bbe1bd..9cfdb2149 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -295,15 +295,15 @@ sudo sntp -sS time.nist.gov ``` ## Connection pool -To handle our connection to the registry, we utilize a connection pool to keep a socket open to increase responsiveness. In order to accomplish this, We are utilizing a heavily modified version of the (geventconnpool)[https://github.com/rasky/geventconnpool] library. +To handle our connection to the registry, we utilize a connection pool to keep a socket open to increase responsiveness. In order to accomplish this, we are utilizing a heavily modified version of the (geventconnpool)[https://github.com/rasky/geventconnpool] library. ### Settings The config for the connection pool exists inside the `settings.py` file. -| Name | Purpose | -| -------- | ------- | -| EPP_CONNECTION_POOL_SIZE | Determines the number of concurrent sockets that should exist in the pool. | -| POOL_KEEP_ALIVE | Determines the interval in which we ping open connections in seconds. Calculated as POOL_KEEP_ALIVE / EPP_CONNECTION_POOL_SIZE | -| POOL_TIMEOUT | Determines how long we try to keep a pool alive for, before restarting it. | +| Name | Purpose | +| ------------------------ | ------------------------------------------------------------------------------------------------- | +| EPP_CONNECTION_POOL_SIZE | Determines the number of concurrent sockets that should exist in the pool. | +| POOL_KEEP_ALIVE | Determines the interval in which we ping open connections in seconds. Calculated as POOL_KEEP_ALIVE / EPP_CONNECTION_POOL_SIZE | +| POOL_TIMEOUT | Determines how long we try to keep a pool alive for, before restarting it. | Consider updating the `POOL_TIMEOUT` or `POOL_KEEP_ALIVE` periods if the pool often restarts. If the pool only restarts after a period of inactivity, update `POOL_KEEP_ALIVE`. If it restarts during the EPP call itself, then `POOL_TIMEOUT` needs to be updated. From a3a623fc98cb9f9d4ec872461aa828207936a733 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:13:01 -0700 Subject: [PATCH 077/119] Update docs/operations/data_migration.md Co-authored-by: Alysia Broddrick <109625347+abroddrick@users.noreply.github.com> --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index fe3d5f45e..aa74d6260 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -423,7 +423,7 @@ Used by the migration scripts to trigger a prompt for deleting all table entries Useful for testing purposes, but *use with caution* ## Import organization data -During MVP, our import scripts did not populate the following fields: `address_line, city, state_territory, and zipcode`. This was primarily due to time constraints. Because of this, we need to run a follow-on script to load this remaining data on each `DomainInformation` object. +During MVP, our import scripts did not populate the following fields: `address_line, city, state_territory, and zipcode` for organization address in Domain Information. This was primarily due to time constraints. Because of this, we need to run a follow-on script to load this remaining data on each `DomainInformation` object. This script is intended to run under the assumption that the [load_transition_domain](#step-1-load-transition-domains) and the [transfer_transition_domains_to_domains](#step-2-transfer-transition-domain-data-into-main-domain-tables) scripts have already been ran. From 4afb230a750c78d426293b66ad8b860ba1657813 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:14:06 -0700 Subject: [PATCH 078/119] Update docs/operations/data_migration.md Co-authored-by: Alysia Broddrick <109625347+abroddrick@users.noreply.github.com> --- docs/operations/data_migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index aa74d6260..058536f85 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -470,6 +470,6 @@ The `load_organization_data` script has five optional parameters. These are as f |:-:|:---------------------------------|:----------------------------------------------------------------------------| | 1 | **sep** | Determines the file separator. Defaults to "\|" | | 2 | **debug** | Increases logging detail. Defaults to False | -| 3 | **directory** | Specifies the containing directory of the data. Defaults to "migrationdata" | +| 3 | **directory** | Specifies the directory containing the files that will be parsed. Defaults to "migrationdata" | | 4 | **domain_additional_filename** | Specifies the filename of domain_additional. Used as an override for the JSON. Has no default. | | 5 | **organization_adhoc_filename** | Specifies the filename of organization_adhoc. Used as an override for the JSON. Has no default. | From 854f0d44644e5125200c08a4920b0695798b4438 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:29:42 -0700 Subject: [PATCH 079/119] Change load_organization_data comment and remove unused params --- .../management/commands/load_organization_data.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 87bc62512..f480bf567 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -1,4 +1,4 @@ -"""Data migration: Send domain invitations once to existing customers.""" +"""Data migration: Load organization data for TransitionDomain and DomainInformation objects""" import argparse import json @@ -54,18 +54,6 @@ class Command(BaseCommand): parser.add_argument("--directory", default="migrationdata", help="Desired directory") - # Serves as a domain_additional_filename override - parser.add_argument( - "--domain_additional_filename", - help="Defines the filename for additional domain data", - ) - - # Serves as a organization_adhoc_filename override - parser.add_argument( - "--organization_adhoc_filename", - help="Defines the filename for domain type adhocs", - ) - def handle(self, migration_json_filename, **options): """Process the objects in TransitionDomain.""" # Parse JSON file From 86506f4e312a9baa2f0175ce8ba5ab70c07ae7c6 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:33:51 -0700 Subject: [PATCH 080/119] Update src/registrar/management/commands/load_organization_data.py Co-authored-by: Alysia Broddrick <109625347+abroddrick@users.noreply.github.com> --- src/registrar/management/commands/load_organization_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index f480bf567..16426eb08 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -55,7 +55,7 @@ class Command(BaseCommand): parser.add_argument("--directory", default="migrationdata", help="Desired directory") def handle(self, migration_json_filename, **options): - """Process the objects in TransitionDomain.""" + """Load organization address data into the TransitionDomain and DomainInformation tables by using the organization adhoc file and domain_additional file""" # Parse JSON file options = self.load_json_settings(options, migration_json_filename) args = TransitionDomainArguments(**options) From 64bd9a38d2b1a63d30899be12d5a64cf86653d05 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:34:16 -0700 Subject: [PATCH 081/119] Update src/registrar/management/commands/utility/extra_transition_domain_helper.py Co-authored-by: Alysia Broddrick <109625347+abroddrick@users.noreply.github.com> --- .../commands/utility/extra_transition_domain_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 75cde995a..a198386fb 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -787,7 +787,7 @@ class OrganizationDataLoader: self.tds_to_update: List[TransitionDomain] = [] def update_organization_data_for_all(self): - """Updates org data for all TransitionDomains""" + """Updates org address data for all TransitionDomains""" all_transition_domains = TransitionDomain.objects.all() if len(all_transition_domains) == 0: raise LoadOrganizationError(code=LoadOrganizationErrorCodes.EMPTY_TRANSITION_DOMAIN_TABLE) From 6da9fdab2abb8f93ab6d3d11c4a95dc0dbc98a0f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Nov 2023 10:36:13 -0500 Subject: [PATCH 082/119] added is_editable to domain; added editable to domain_detail.html; added editable to domain_sidebar.html; added editable to summary_item.html; updated has_permission in domain views to check for domain editable --- src/registrar/models/domain.py | 8 ++++++++ src/registrar/templates/domain_detail.html | 14 +++++++------- src/registrar/templates/domain_sidebar.html | 2 ++ src/registrar/templates/includes/summary_item.html | 2 +- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 37e78ec6e..93b50bf81 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -880,6 +880,14 @@ class Domain(TimeStampedModel, DomainHelper): """ return self.state == self.State.READY + def is_editable(self) -> bool: + """domain is editable unless state is on hold or deleted""" + return self.state in [ + self.State.UNKNOWN, + self.State.DNS_NEEDED, + self.State.READY, + ] + def transfer(self): """Going somewhere. Not implemented.""" raise NotImplementedError() diff --git a/src/registrar/templates/domain_detail.html b/src/registrar/templates/domain_detail.html index e220fe1aa..2bd6aa0ed 100644 --- a/src/registrar/templates/domain_detail.html +++ b/src/registrar/templates/domain_detail.html @@ -29,7 +29,7 @@ {% url 'domain-dns-nameservers' pk=domain.id as url %} {% if domain.nameservers|length > 0 %} - {% include "includes/summary_item.html" with title='DNS name servers' domains='true' value=domain.nameservers list='true' edit_link=url %} + {% include "includes/summary_item.html" with title='DNS name servers' domains='true' value=domain.nameservers list='true' edit_link=url editable=domain.is_editable %} {% else %}

    DNS name servers

    No DNS name servers have been added yet. Before your domain can be used we’ll need information about your domain name servers.

    @@ -37,22 +37,22 @@ {% endif %} {% url 'domain-org-name-address' pk=domain.id as url %} - {% include "includes/summary_item.html" with title='Organization name and mailing address' value=domain.domain_info address='true' edit_link=url %} + {% include "includes/summary_item.html" with title='Organization name and mailing address' value=domain.domain_info address='true' edit_link=url editable=domain.is_editable %} {% url 'domain-authorizing-official' pk=domain.id as url %} - {% include "includes/summary_item.html" with title='Authorizing official' value=domain.domain_info.authorizing_official contact='true' edit_link=url %} + {% include "includes/summary_item.html" with title='Authorizing official' value=domain.domain_info.authorizing_official contact='true' edit_link=url editable=domain.is_editable %} {% url 'domain-your-contact-information' pk=domain.id as url %} - {% include "includes/summary_item.html" with title='Your contact information' value=request.user.contact contact='true' edit_link=url %} + {% include "includes/summary_item.html" with title='Your contact information' value=request.user.contact contact='true' edit_link=url editable=domain.is_editable %} {% url 'domain-security-email' pk=domain.id as url %} {% if security_email is not None and security_email != default_security_email%} - {% include "includes/summary_item.html" with title='Security email' value=security_email edit_link=url %} + {% include "includes/summary_item.html" with title='Security email' value=security_email edit_link=url editable=domain.is_editable %} {% else %} - {% include "includes/summary_item.html" with title='Security email' value='None provided' edit_link=url %} + {% include "includes/summary_item.html" with title='Security email' value='None provided' edit_link=url editable=domain.is_editable %} {% endif %} {% url 'domain-users' pk=domain.id as url %} - {% include "includes/summary_item.html" with title='Domain managers' users='true' list=True value=domain.permissions.all edit_link=url %} + {% include "includes/summary_item.html" with title='Domain managers' users='true' list=True value=domain.permissions.all edit_link=url editable=domain.is_editable %} {% endblock %} {# domain_content #} diff --git a/src/registrar/templates/domain_sidebar.html b/src/registrar/templates/domain_sidebar.html index 6be8f655d..c224d60c1 100644 --- a/src/registrar/templates/domain_sidebar.html +++ b/src/registrar/templates/domain_sidebar.html @@ -12,6 +12,7 @@
    • + {% if domain.is_editable %}
  • {% url 'domain-dns' pk=domain.id as url %} @@ -98,6 +99,7 @@ Domain managers
    • + {% endif %} diff --git a/src/registrar/templates/includes/summary_item.html b/src/registrar/templates/includes/summary_item.html index 8a33bb1d5..dea14553b 100644 --- a/src/registrar/templates/includes/summary_item.html +++ b/src/registrar/templates/includes/summary_item.html @@ -85,7 +85,7 @@ {% endif %} - {% if edit_link %} + {% if editable and edit_link %}
    Date: Wed, 22 Nov 2023 08:37:01 -0700 Subject: [PATCH 083/119] Update src/registrar/management/commands/utility/extra_transition_domain_helper.py Co-authored-by: Alysia Broddrick <109625347+abroddrick@users.noreply.github.com> --- .../commands/utility/extra_transition_domain_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index a198386fb..3f6b2423f 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -845,7 +845,7 @@ class OrganizationDataLoader: def parse_org_data(self, domain_name, transition_domain: TransitionDomain) -> TransitionDomain: """Grabs organization_name from the parsed files and associates it - with a transition_domain object, then returns that object.""" + with a transition_domain object, then updates that transition domain object and returns it""" if not isinstance(transition_domain, TransitionDomain): raise ValueError("Not a valid object, must be TransitionDomain") From 99d449679cc0c709f709f380886559c596a45954 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:37:20 -0700 Subject: [PATCH 084/119] Update src/registrar/management/commands/load_organization_data.py Co-authored-by: Alysia Broddrick <109625347+abroddrick@users.noreply.github.com> --- src/registrar/management/commands/load_organization_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 16426eb08..bc4d6698b 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -97,7 +97,7 @@ class Command(BaseCommand): Number of DomainInformation objects to (potentially) change: {len(transition_domains)} For each DomainInformation, modify the following fields: {self.changed_fields} """, - prompt_title="Do you wish to load organization data for DomainInformation?", + prompt_title="Do you wish to update organization address data for DomainInformation as well?", ) logger.info( From 8535d783e82094bb787b2c771a7a109c0ae95ba9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:37:48 -0700 Subject: [PATCH 085/119] PR suggestions Change variable names and coments --- .../commands/load_organization_data.py | 22 +++++++++---------- .../utility/extra_transition_domain_helper.py | 2 ++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index f480bf567..087baaf17 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -58,26 +58,26 @@ class Command(BaseCommand): """Process the objects in TransitionDomain.""" # Parse JSON file options = self.load_json_settings(options, migration_json_filename) - args = TransitionDomainArguments(**options) + org_args = TransitionDomainArguments(**options) # Will sys.exit() when prompt is "n" TerminalHelper.prompt_for_execution( system_exit_on_terminate=True, info_to_inspect=f""" ==Master data file== - domain_additional_filename: {args.domain_additional_filename} + domain_additional_filename: {org_args.domain_additional_filename} ==Organization data== - organization_adhoc_filename: {args.organization_adhoc_filename} + organization_adhoc_filename: {org_args.organization_adhoc_filename} ==Containing directory== - directory: {args.directory} + directory: {org_args.directory} """, prompt_title="Do you wish to load organization data for TransitionDomains?", ) - load = OrganizationDataLoader(args) - transition_domains = load.update_organization_data_for_all() + org_load_helper = OrganizationDataLoader(org_args) + transition_domains = org_load_helper.update_organization_data_for_all() # Reprompt the user to reinspect before updating DomainInformation # Will sys.exit() when prompt is "n" @@ -85,13 +85,13 @@ class Command(BaseCommand): system_exit_on_terminate=True, info_to_inspect=f""" ==Master data file== - domain_additional_filename: {args.domain_additional_filename} + domain_additional_filename: {org_args.domain_additional_filename} ==Organization name information== - organization_adhoc_filename: {args.organization_adhoc_filename} + organization_adhoc_filename: {org_args.organization_adhoc_filename} ==Containing directory== - directory: {args.directory} + directory: {org_args.directory} ==Proposed Changes== Number of DomainInformation objects to (potentially) change: {len(transition_domains)} @@ -105,10 +105,10 @@ class Command(BaseCommand): "Preparing to load organization data onto DomainInformation tables..." f"{TerminalColors.ENDC}" ) - self.prepare_update_domain_information(transition_domains, args.debug) + self.prepare_update_domain_information(transition_domains, org_args.debug) logger.info(f"{TerminalColors.MAGENTA}" f"Beginning mass DomainInformation update..." f"{TerminalColors.ENDC}") - self.bulk_update_domain_information(args.debug) + self.bulk_update_domain_information(org_args.debug) def load_json_settings(self, options, migration_json_filename): """Parses options from the given JSON file.""" diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index 75cde995a..677885839 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -800,6 +800,8 @@ class OrganizationDataLoader: return self.tds_to_update def prepare_transition_domains(self, transition_domains): + """Pares org data for each transition domain, + then appends it to the tds_to_update list""" for item in transition_domains: updated = self.parse_org_data(item.domain_name, item) self.tds_to_update.append(updated) From 1efb7d23f3a161703b71fd1e9336735c0d35fcd2 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Nov 2023 10:48:13 -0500 Subject: [PATCH 086/119] domain views has_permissions updated --- src/registrar/views/domain.py | 22 ++++++++++++++++++++++ src/registrar/views/utility/mixins.py | 17 +++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 0b950ba7a..a9d0d4510 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -152,6 +152,28 @@ class DomainView(DomainBaseView): context["security_email"] = security_email return context + def in_editable_state(self, pk): + """Override in_editable_state from DomainPermission + Allow detail page to be editable""" + + requested_domain = None + if Domain.objects.filter(id=pk).exists(): + requested_domain = Domain.objects.get(id=pk) + + # if domain is editable return true + if requested_domain: + return True + return False + + def _get_domain(self, request): + """ + override get_domain for this view so that domain overview + always resets the cache for the domain object + """ + self.session = request.session + self.object = self.get_object() + self._update_session_with_domain() + class DomainOrgNameAddressView(DomainFormBaseView): """Organization name and mailing address view""" diff --git a/src/registrar/views/utility/mixins.py b/src/registrar/views/utility/mixins.py index e37ff4927..596873cf3 100644 --- a/src/registrar/views/utility/mixins.py +++ b/src/registrar/views/utility/mixins.py @@ -3,6 +3,7 @@ from django.contrib.auth.mixins import PermissionRequiredMixin from registrar.models import ( + Domain, DomainApplication, DomainInvitation, DomainInformation, @@ -52,9 +53,25 @@ class DomainPermission(PermissionsLoginMixin): if not UserDomainRole.objects.filter(user=self.request.user, domain__id=pk).exists(): return False + # test if domain in editable state + if not self.in_editable_state(pk): + return False + # if we need to check more about the nature of role, do it here. return True + def in_editable_state(self, pk): + """Is the domain in an editable state""" + + requested_domain = None + if Domain.objects.filter(id=pk).exists(): + requested_domain = Domain.objects.get(id=pk) + + # if domain is editable return true + if requested_domain and requested_domain.is_editable(): + return True + return False + def can_access_other_user_domains(self, pk): """Checks to see if an authorized user (staff or superuser) can access a domain that they did not create or was invited to. From db964498e1c8edc231b4e8267df0f618df64537f Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 08:49:48 -0700 Subject: [PATCH 087/119] Linting --- src/registrar/management/commands/load_organization_data.py | 3 ++- .../commands/utility/extra_transition_domain_helper.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 0036faf45..618e700b0 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -55,7 +55,8 @@ class Command(BaseCommand): parser.add_argument("--directory", default="migrationdata", help="Desired directory") def handle(self, migration_json_filename, **options): - """Load organization address data into the TransitionDomain and DomainInformation tables by using the organization adhoc file and domain_additional file""" + """Load organization address data into the TransitionDomain + and DomainInformation tables by using the organization adhoc file and domain_additional file""" # Parse JSON file options = self.load_json_settings(options, migration_json_filename) org_args = TransitionDomainArguments(**options) diff --git a/src/registrar/management/commands/utility/extra_transition_domain_helper.py b/src/registrar/management/commands/utility/extra_transition_domain_helper.py index ea3f3911f..04170811f 100644 --- a/src/registrar/management/commands/utility/extra_transition_domain_helper.py +++ b/src/registrar/management/commands/utility/extra_transition_domain_helper.py @@ -800,7 +800,7 @@ class OrganizationDataLoader: return self.tds_to_update def prepare_transition_domains(self, transition_domains): - """Pares org data for each transition domain, + """Parses org data for each transition domain, then appends it to the tds_to_update list""" for item in transition_domains: updated = self.parse_org_data(item.domain_name, item) From 91d33f4c2a4da291aaee45049f006b59bcf8010c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 22 Nov 2023 09:08:49 -0700 Subject: [PATCH 088/119] Linting --- src/registrar/management/commands/load_organization_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/management/commands/load_organization_data.py b/src/registrar/management/commands/load_organization_data.py index 618e700b0..122795400 100644 --- a/src/registrar/management/commands/load_organization_data.py +++ b/src/registrar/management/commands/load_organization_data.py @@ -55,7 +55,7 @@ class Command(BaseCommand): parser.add_argument("--directory", default="migrationdata", help="Desired directory") def handle(self, migration_json_filename, **options): - """Load organization address data into the TransitionDomain + """Load organization address data into the TransitionDomain and DomainInformation tables by using the organization adhoc file and domain_additional file""" # Parse JSON file options = self.load_json_settings(options, migration_json_filename) From 3ec910c37a7a9c8c651750ccd9a2b66b6b3cb529 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Nov 2023 11:18:54 -0500 Subject: [PATCH 089/119] wrote unit tests --- src/registrar/tests/test_views.py | 80 +++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 936c344f7..5b9ffa4bb 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1082,6 +1082,7 @@ class TestWithDomainPermissions(TestWithUser): self.domain_with_ip, _ = Domain.objects.get_or_create(name="nameserverwithip.gov") self.domain_just_nameserver, _ = Domain.objects.get_or_create(name="justnameserver.com") self.domain_no_information, _ = Domain.objects.get_or_create(name="noinformation.gov") + self.domain_on_hold, _ = Domain.objects.get_or_create(name="on-hold.gov", state=Domain.State.ON_HOLD) self.domain_dsdata, _ = Domain.objects.get_or_create(name="dnssec-dsdata.gov") self.domain_multdsdata, _ = Domain.objects.get_or_create(name="dnssec-multdsdata.gov") @@ -1096,6 +1097,7 @@ class TestWithDomainPermissions(TestWithUser): DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_dnssec_none) DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_with_ip) DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_just_nameserver) + DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_on_hold) self.role, _ = UserDomainRole.objects.get_or_create( user=self.user, domain=self.domain, role=UserDomainRole.Roles.MANAGER @@ -1124,6 +1126,9 @@ class TestWithDomainPermissions(TestWithUser): domain=self.domain_just_nameserver, role=UserDomainRole.Roles.MANAGER, ) + UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_on_hold, role=UserDomainRole.Roles.MANAGER + ) def tearDown(self): try: @@ -1204,6 +1209,15 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): response = self.client.get(reverse("domain", kwargs={"pk": self.domain.id})) self.assertEqual(response.status_code, 403) + def test_domain_overview_allowed_for_on_hold(self): + """Test that the domain overview page displays for on hold domain""" + home_page = self.app.get("/") + self.assertContains(home_page, "on-hold.gov") + + # View domain overview page + detail_page = self.client.get(reverse("domain", kwargs={"pk": self.domain_on_hold.id})) + self.assertNotContains(detail_page, "Edit") + def test_domain_see_just_nameserver(self): home_page = self.app.get("/") self.assertContains(home_page, "justnameserver.com") @@ -1258,6 +1272,19 @@ class TestDomainManagers(TestDomainOverview): response = self.client.get(reverse("domain-users", kwargs={"pk": self.domain.id})) self.assertContains(response, "Domain managers") + def test_domain_users_blocked_for_on_hold(self): + """Test that the domain users page blocked for on hold domain""" + + # attempt to view domain users page + with less_console_noise(): + response = self.client.get(reverse("domain-users", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + + # attempt to view domain users add page + with less_console_noise(): + response = self.client.get(reverse("domain-users-add", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + def test_domain_managers_add_link(self): """Button to get to user add page works.""" management_page = self.app.get(reverse("domain-users", kwargs={"pk": self.domain.id})) @@ -1391,6 +1418,14 @@ class TestDomainNameservers(TestDomainOverview): page = self.client.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain.id})) self.assertContains(page, "DNS name servers") + def test_domain_nameservers_blocked_for_on_hold(self): + """Test that the domain nameservers page blocked for on hold domain""" + + # attempt to view domain nameservers page + with less_console_noise(): + response = self.client.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + def test_domain_nameservers_form_submit_one_nameserver(self): """Nameserver form submitted with one nameserver throws error. @@ -1606,6 +1641,14 @@ class TestDomainAuthorizingOfficial(TestDomainOverview): # once on the sidebar, once in the title self.assertContains(page, "Authorizing official", count=2) + def test_domain_authorizing_official_blocked_for_on_hold(self): + """Test that the domain authorizing official page blocked for on hold domain""" + + # attempt to view domain authorizing official page + with less_console_noise(): + response = self.client.get(reverse("domain-authorizing-official", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + def test_domain_authorizing_official_content(self): """Authorizing official information appears on the page.""" self.domain_information.authorizing_official = Contact(first_name="Testy") @@ -1622,6 +1665,14 @@ class TestDomainOrganization(TestDomainOverview): # once on the sidebar, once in the page title, once as H1 self.assertContains(page, "Organization name and mailing address", count=3) + def test_domain_org_name_blocked_for_on_hold(self): + """Test that the domain org name page blocked for on hold domain""" + + # attempt to view domain org name page + with less_console_noise(): + response = self.client.get(reverse("domain-org-name-address", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + def test_domain_org_name_address_content(self): """Org name and address information appears on the page.""" self.domain_information.organization_name = "Town of Igorville" @@ -1653,6 +1704,14 @@ class TestDomainContactInformation(TestDomainOverview): page = self.client.get(reverse("domain-your-contact-information", kwargs={"pk": self.domain.id})) self.assertContains(page, "Your contact information") + def test_domain_contact_information_blocked_for_on_hold(self): + """Test that the domain contact information page blocked for on hold domain""" + + # attempt to view domain contact information page + with less_console_noise(): + response = self.client.get(reverse("domain-your-contact-information", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + def test_domain_your_contact_information_content(self): """Logged-in user's contact information appears on the page.""" self.user.contact.first_name = "Testy" @@ -1678,6 +1737,14 @@ class TestDomainSecurityEmail(TestDomainOverview): self.assertContains(page, "security@mail.gov") self.mockSendPatch.stop() + def test_domain_security_email_blocked_for_on_hold(self): + """Test that the domain security email page blocked for on hold domain""" + + # attempt to view domain security email page + with less_console_noise(): + response = self.client.get(reverse("domain-security-email", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + def test_domain_security_email_no_security_contact(self): """Loads a domain with no defined security email. We should not show the default.""" @@ -1805,6 +1872,19 @@ class TestDomainDNSSEC(TestDomainOverview): page = self.client.get(reverse("domain-dns-dnssec", kwargs={"pk": self.domain.id})) self.assertContains(page, "Enable DNSSEC") + def test_domain_dnssec_blocked_for_on_hold(self): + """Test that the domain dnssec page blocked for on hold domain""" + + # attempt to view domain dnssec page + with less_console_noise(): + response = self.client.get(reverse("domain-dns-dnssec", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + + # attempt to view domain dnssec dsdata page + with less_console_noise(): + response = self.client.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_on_hold.id})) + self.assertEqual(response.status_code, 403) + def test_dnssec_page_loads_with_data_in_domain(self): """DNSSEC overview page loads when domain has DNSSEC data and the template contains a button to disable DNSSEC.""" From 570c5c3020f148e1732a9d0a49bfc4c6cc919698 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Nov 2023 11:50:18 -0500 Subject: [PATCH 090/119] formatted for linter --- src/registrar/tests/test_views.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 5b9ffa4bb..8818ae94f 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1213,11 +1213,11 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): """Test that the domain overview page displays for on hold domain""" home_page = self.app.get("/") self.assertContains(home_page, "on-hold.gov") - + # View domain overview page detail_page = self.client.get(reverse("domain", kwargs={"pk": self.domain_on_hold.id})) self.assertNotContains(detail_page, "Edit") - + def test_domain_see_just_nameserver(self): home_page = self.app.get("/") self.assertContains(home_page, "justnameserver.com") @@ -1274,7 +1274,7 @@ class TestDomainManagers(TestDomainOverview): def test_domain_users_blocked_for_on_hold(self): """Test that the domain users page blocked for on hold domain""" - + # attempt to view domain users page with less_console_noise(): response = self.client.get(reverse("domain-users", kwargs={"pk": self.domain_on_hold.id})) @@ -1420,7 +1420,7 @@ class TestDomainNameservers(TestDomainOverview): def test_domain_nameservers_blocked_for_on_hold(self): """Test that the domain nameservers page blocked for on hold domain""" - + # attempt to view domain nameservers page with less_console_noise(): response = self.client.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain_on_hold.id})) @@ -1643,7 +1643,7 @@ class TestDomainAuthorizingOfficial(TestDomainOverview): def test_domain_authorizing_official_blocked_for_on_hold(self): """Test that the domain authorizing official page blocked for on hold domain""" - + # attempt to view domain authorizing official page with less_console_noise(): response = self.client.get(reverse("domain-authorizing-official", kwargs={"pk": self.domain_on_hold.id})) @@ -1667,7 +1667,7 @@ class TestDomainOrganization(TestDomainOverview): def test_domain_org_name_blocked_for_on_hold(self): """Test that the domain org name page blocked for on hold domain""" - + # attempt to view domain org name page with less_console_noise(): response = self.client.get(reverse("domain-org-name-address", kwargs={"pk": self.domain_on_hold.id})) @@ -1706,10 +1706,12 @@ class TestDomainContactInformation(TestDomainOverview): def test_domain_contact_information_blocked_for_on_hold(self): """Test that the domain contact information page blocked for on hold domain""" - + # attempt to view domain contact information page with less_console_noise(): - response = self.client.get(reverse("domain-your-contact-information", kwargs={"pk": self.domain_on_hold.id})) + response = self.client.get( + reverse("domain-your-contact-information", kwargs={"pk": self.domain_on_hold.id}) + ) self.assertEqual(response.status_code, 403) def test_domain_your_contact_information_content(self): @@ -1739,7 +1741,7 @@ class TestDomainSecurityEmail(TestDomainOverview): def test_domain_security_email_blocked_for_on_hold(self): """Test that the domain security email page blocked for on hold domain""" - + # attempt to view domain security email page with less_console_noise(): response = self.client.get(reverse("domain-security-email", kwargs={"pk": self.domain_on_hold.id})) @@ -1874,7 +1876,7 @@ class TestDomainDNSSEC(TestDomainOverview): def test_domain_dnssec_blocked_for_on_hold(self): """Test that the domain dnssec page blocked for on hold domain""" - + # attempt to view domain dnssec page with less_console_noise(): response = self.client.get(reverse("domain-dns-dnssec", kwargs={"pk": self.domain_on_hold.id})) @@ -1884,7 +1886,7 @@ class TestDomainDNSSEC(TestDomainOverview): with less_console_noise(): response = self.client.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_on_hold.id})) self.assertEqual(response.status_code, 403) - + def test_dnssec_page_loads_with_data_in_domain(self): """DNSSEC overview page loads when domain has DNSSEC data and the template contains a button to disable DNSSEC.""" From 8300694a3a63760dc478f69fa605a0a776c7dd6a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Nov 2023 14:26:24 -0500 Subject: [PATCH 091/119] added test cases for deleted state; updated fetch_cache logic to properly handle deleted domains --- src/registrar/models/domain.py | 2 +- src/registrar/tests/test_views.py | 98 ++++++++++--------------------- 2 files changed, 31 insertions(+), 69 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 93b50bf81..d28008c9a 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -1196,7 +1196,7 @@ class Domain(TimeStampedModel, DomainHelper): logger.error(e) logger.error(e.code) raise e - if e.code == ErrorCode.OBJECT_DOES_NOT_EXIST: + if e.code == ErrorCode.OBJECT_DOES_NOT_EXIST and self.state != Domain.State.DELETED: # avoid infinite loop already_tried_to_create = True self.dns_needed_from_unknown() diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 8818ae94f..d6bad9cdf 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1083,6 +1083,7 @@ class TestWithDomainPermissions(TestWithUser): self.domain_just_nameserver, _ = Domain.objects.get_or_create(name="justnameserver.com") self.domain_no_information, _ = Domain.objects.get_or_create(name="noinformation.gov") self.domain_on_hold, _ = Domain.objects.get_or_create(name="on-hold.gov", state=Domain.State.ON_HOLD) + self.domain_deleted, _ = Domain.objects.get_or_create(name="deleted.gov", state=Domain.State.DELETED) self.domain_dsdata, _ = Domain.objects.get_or_create(name="dnssec-dsdata.gov") self.domain_multdsdata, _ = Domain.objects.get_or_create(name="dnssec-multdsdata.gov") @@ -1098,6 +1099,7 @@ class TestWithDomainPermissions(TestWithUser): DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_with_ip) DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_just_nameserver) DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_on_hold) + DomainInformation.objects.get_or_create(creator=self.user, domain=self.domain_deleted) self.role, _ = UserDomainRole.objects.get_or_create( user=self.user, domain=self.domain, role=UserDomainRole.Roles.MANAGER @@ -1129,6 +1131,9 @@ class TestWithDomainPermissions(TestWithUser): UserDomainRole.objects.get_or_create( user=self.user, domain=self.domain_on_hold, role=UserDomainRole.Roles.MANAGER ) + UserDomainRole.objects.get_or_create( + user=self.user, domain=self.domain_deleted, role=UserDomainRole.Roles.MANAGER + ) def tearDown(self): try: @@ -1182,6 +1187,31 @@ class TestDomainPermissions(TestWithDomainPermissions): response = self.client.get(reverse(view_name, kwargs={"pk": self.domain.id})) self.assertEqual(response.status_code, 403) + def test_domain_pages_blocked_for_on_hold_and_deleted(self): + """Test that the domain pages are blocked for on hold and deleted domains""" + + self.client.force_login(self.user) + for view_name in [ + "domain-users", + "domain-users-add", + "domain-dns", + "domain-dns-nameservers", + "domain-dns-dnssec", + "domain-dns-dnssec-dsdata", + "domain-org-name-address", + "domain-authorizing-official", + "domain-your-contact-information", + "domain-security-email", + ]: + for domain in [ + self.domain_on_hold, + self.domain_deleted, + ]: + with self.subTest(view_name=view_name, domain=domain): + with less_console_noise(): + response = self.client.get(reverse(view_name, kwargs={"pk": domain.id})) + self.assertEqual(response.status_code, 403) + class TestDomainOverview(TestWithDomainPermissions, WebTest): def setUp(self): @@ -1272,19 +1302,6 @@ class TestDomainManagers(TestDomainOverview): response = self.client.get(reverse("domain-users", kwargs={"pk": self.domain.id})) self.assertContains(response, "Domain managers") - def test_domain_users_blocked_for_on_hold(self): - """Test that the domain users page blocked for on hold domain""" - - # attempt to view domain users page - with less_console_noise(): - response = self.client.get(reverse("domain-users", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - - # attempt to view domain users add page - with less_console_noise(): - response = self.client.get(reverse("domain-users-add", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - def test_domain_managers_add_link(self): """Button to get to user add page works.""" management_page = self.app.get(reverse("domain-users", kwargs={"pk": self.domain.id})) @@ -1418,14 +1435,6 @@ class TestDomainNameservers(TestDomainOverview): page = self.client.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain.id})) self.assertContains(page, "DNS name servers") - def test_domain_nameservers_blocked_for_on_hold(self): - """Test that the domain nameservers page blocked for on hold domain""" - - # attempt to view domain nameservers page - with less_console_noise(): - response = self.client.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - def test_domain_nameservers_form_submit_one_nameserver(self): """Nameserver form submitted with one nameserver throws error. @@ -1641,14 +1650,6 @@ class TestDomainAuthorizingOfficial(TestDomainOverview): # once on the sidebar, once in the title self.assertContains(page, "Authorizing official", count=2) - def test_domain_authorizing_official_blocked_for_on_hold(self): - """Test that the domain authorizing official page blocked for on hold domain""" - - # attempt to view domain authorizing official page - with less_console_noise(): - response = self.client.get(reverse("domain-authorizing-official", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - def test_domain_authorizing_official_content(self): """Authorizing official information appears on the page.""" self.domain_information.authorizing_official = Contact(first_name="Testy") @@ -1665,14 +1666,6 @@ class TestDomainOrganization(TestDomainOverview): # once on the sidebar, once in the page title, once as H1 self.assertContains(page, "Organization name and mailing address", count=3) - def test_domain_org_name_blocked_for_on_hold(self): - """Test that the domain org name page blocked for on hold domain""" - - # attempt to view domain org name page - with less_console_noise(): - response = self.client.get(reverse("domain-org-name-address", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - def test_domain_org_name_address_content(self): """Org name and address information appears on the page.""" self.domain_information.organization_name = "Town of Igorville" @@ -1704,16 +1697,6 @@ class TestDomainContactInformation(TestDomainOverview): page = self.client.get(reverse("domain-your-contact-information", kwargs={"pk": self.domain.id})) self.assertContains(page, "Your contact information") - def test_domain_contact_information_blocked_for_on_hold(self): - """Test that the domain contact information page blocked for on hold domain""" - - # attempt to view domain contact information page - with less_console_noise(): - response = self.client.get( - reverse("domain-your-contact-information", kwargs={"pk": self.domain_on_hold.id}) - ) - self.assertEqual(response.status_code, 403) - def test_domain_your_contact_information_content(self): """Logged-in user's contact information appears on the page.""" self.user.contact.first_name = "Testy" @@ -1739,14 +1722,6 @@ class TestDomainSecurityEmail(TestDomainOverview): self.assertContains(page, "security@mail.gov") self.mockSendPatch.stop() - def test_domain_security_email_blocked_for_on_hold(self): - """Test that the domain security email page blocked for on hold domain""" - - # attempt to view domain security email page - with less_console_noise(): - response = self.client.get(reverse("domain-security-email", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - def test_domain_security_email_no_security_contact(self): """Loads a domain with no defined security email. We should not show the default.""" @@ -1874,19 +1849,6 @@ class TestDomainDNSSEC(TestDomainOverview): page = self.client.get(reverse("domain-dns-dnssec", kwargs={"pk": self.domain.id})) self.assertContains(page, "Enable DNSSEC") - def test_domain_dnssec_blocked_for_on_hold(self): - """Test that the domain dnssec page blocked for on hold domain""" - - # attempt to view domain dnssec page - with less_console_noise(): - response = self.client.get(reverse("domain-dns-dnssec", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - - # attempt to view domain dnssec dsdata page - with less_console_noise(): - response = self.client.get(reverse("domain-dns-dnssec-dsdata", kwargs={"pk": self.domain_on_hold.id})) - self.assertEqual(response.status_code, 403) - def test_dnssec_page_loads_with_data_in_domain(self): """DNSSEC overview page loads when domain has DNSSEC data and the template contains a button to disable DNSSEC.""" From 441a6027baf23d180792eab6d699b22cb978390b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Wed, 22 Nov 2023 17:41:27 -0500 Subject: [PATCH 092/119] expiration date written to db on fetch_cache --- src/registrar/models/domain.py | 10 +++++++--- src/registrar/tests/common.py | 1 + src/registrar/tests/test_models_domain.py | 7 +++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 37e78ec6e..4cb2e72a3 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -214,9 +214,7 @@ class Domain(TimeStampedModel, DomainHelper): """Get or set the `ex_date` element from the registry. Additionally, update the expiration date in the registrar""" try: - self.expiration_date = self._get_property("ex_date") - self.save() - return self.expiration_date + return self._get_property("ex_date") except Exception as e: # exception raised during the save to registrar logger.error(f"error updating expiration date in registrar: {e}") @@ -1602,6 +1600,12 @@ class Domain(TimeStampedModel, DomainHelper): if old_cache_contacts is not None: cleaned["contacts"] = old_cache_contacts + # if expiration date from registry does not match what is in db, + # update the db + if "ex_date" in cleaned and cleaned["ex_date"] != self.expiration_date: + self.expiration_date = cleaned["ex_date"] + self.save() + self._cache = cleaned except RegistryError as e: diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 9a062106f..46081e98f 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -617,6 +617,7 @@ class MockEppLib(TestCase): common.Status(state="serverTransferProhibited", description="", lang="en"), common.Status(state="inactive", description="", lang="en"), ], + ex_date=datetime.date(2023, 5, 25), ) mockDataInfoContact = mockDataInfoDomain.dummyInfoContactResultData( "123", "123@mail.gov", datetime.datetime(2023, 5, 25, 19, 45, 35), "lastPw" diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index c75b1b935..4a2023243 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -1987,6 +1987,13 @@ class TestExpirationDate(MockEppLib): with self.assertRaises(RegistryError): self.domain_w_error.renew_domain() + def test_expiration_date_updated_on_info_domain_call(self): + """assert that expiration date in db is updated on info domain call""" + # force fetch_cache to be called + self.domain.statuses + test_date = datetime.date(2023, 5, 25) + self.assertEquals(self.domain.expiration_date, test_date) + class TestAnalystClientHold(MockEppLib): """Rule: Analysts may suspend or restore a domain by using client hold""" From 46677f4588458d7aa8c8157de17eef79888320bd Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 24 Nov 2023 07:43:25 -0700 Subject: [PATCH 093/119] Add filter + test skeleton --- src/registrar/admin.py | 4 ++++ src/registrar/tests/test_admin.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index c059e5674..9921b1194 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -356,6 +356,10 @@ class DomainInvitationAdmin(ListHeaderAdmin): "email", "domain__name", ] + + # Filters + list_filter = ("status",) + search_help_text = "Search by email or domain." # Mark the FSM field 'status' as readonly diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index a7cbb5d33..1ad5f95da 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -8,6 +8,7 @@ from registrar.admin import ( DomainAdmin, DomainApplicationAdmin, DomainApplicationAdminForm, + DomainInvitationAdmin, ListHeaderAdmin, MyUserAdmin, AuditedAdmin, @@ -847,6 +848,42 @@ class TestDomainApplicationAdmin(MockEppLib): User.objects.all().delete() +class DomainInvitationAdminTest(TestCase): + def setUp(self): + self.site = AdminSite() + self.factory = RequestFactory() + self.admin = ListHeaderAdmin(model=DomainInvitationAdmin, admin_site=None) + self.client = Client(HTTP_HOST="localhost:8080") + self.superuser = create_superuser() + + def tearDown(self): + # delete any applications too + DomainInvitation.objects.all().delete() + DomainApplication.objects.all().delete() + User.objects.all().delete() + + def test_get_filters(self): + # Create a mock request object + request = self.factory.get("/admin/yourmodel/") + # Set the GET parameters for testing + request.GET = { + "status": "started", + "investigator": "Rachid Mrad", + "q": "search_value", + } + # Call the get_filters method + filters = self.admin.get_filters(request) + + # Assert the filters extracted from the request GET + self.assertEqual( + filters, + [ + {"parameter_name": "status", "parameter_value": "started"}, + {"parameter_name": "investigator", "parameter_value": "Rachid Mrad"}, + ], + ) + + class ListHeaderAdminTest(TestCase): def setUp(self): self.site = AdminSite() From 377786cccfbccfc3e55f551b20cd469eee457522 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 24 Nov 2023 14:16:34 -0700 Subject: [PATCH 094/119] Add test cases --- src/registrar/admin.py | 5 +++++ src/registrar/tests/test_admin.py | 34 ++++++++++++++----------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index 9921b1194..ff0f71426 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -343,6 +343,11 @@ class UserDomainRoleAdmin(ListHeaderAdmin): class DomainInvitationAdmin(ListHeaderAdmin): """Custom domain invitation admin class.""" + class Meta: + model = models.DomainInvitation + fields = "__all__" + + _meta = Meta() # Columns list_display = [ diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 1ad5f95da..089e569eb 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -850,9 +850,8 @@ class TestDomainApplicationAdmin(MockEppLib): class DomainInvitationAdminTest(TestCase): def setUp(self): - self.site = AdminSite() self.factory = RequestFactory() - self.admin = ListHeaderAdmin(model=DomainInvitationAdmin, admin_site=None) + self.admin = ListHeaderAdmin(model=DomainInvitationAdmin, admin_site=AdminSite()) self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() @@ -863,26 +862,23 @@ class DomainInvitationAdminTest(TestCase): User.objects.all().delete() def test_get_filters(self): - # Create a mock request object - request = self.factory.get("/admin/yourmodel/") - # Set the GET parameters for testing - request.GET = { - "status": "started", - "investigator": "Rachid Mrad", - "q": "search_value", - } - # Call the get_filters method - filters = self.admin.get_filters(request) + # Have to get creative to get past linter + p = "adminpass" + self.client.login(username="superuser", password=p) - # Assert the filters extracted from the request GET - self.assertEqual( - filters, - [ - {"parameter_name": "status", "parameter_value": "started"}, - {"parameter_name": "investigator", "parameter_value": "Rachid Mrad"}, - ], + # Mock a user + user = mock_user() + + response = self.client.get( + "/admin/registrar/domaininvitation/", + {}, + follow=True, ) + # Assert that the filters are added + self.assertContains(response, "invited", count=4) + self.assertContains(response, "retrieved", count=4) + class ListHeaderAdminTest(TestCase): def setUp(self): From 7bac5185b07af4b2ba1463186aa0a4ad838f6fa1 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 25 Nov 2023 06:06:37 -0500 Subject: [PATCH 095/119] updated check availability message in api and in form; modified js to display html rather than text --- src/api/views.py | 10 ++++++++-- src/registrar/assets/js/get-gov.js | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/api/views.py b/src/api/views.py index 2cb23a9b2..5e5365e58 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -2,6 +2,9 @@ from django.apps import apps from django.views.decorators.http import require_http_methods from django.http import JsonResponse +from django.utils.safestring import mark_safe + +from registrar.templatetags.url_helpers import public_site_url import requests @@ -18,8 +21,11 @@ DOMAIN_API_MESSAGES = { " For example, if you want www.city.gov, you would enter “city”" " (without the quotes).", "extra_dots": "Enter the .gov domain you want without any periods.", - "unavailable": "That domain isn’t available. Try entering another one." - " Contact us if you need help coming up with a domain.", + "unavailable": mark_safe( # nosec + "That domain isn’t available. " + "" + "Read more about choosing your .gov domain.".format(public_site_url("domains/choosing")) + ), "invalid": "Enter a domain using only letters, numbers, or hyphens (though we don't recommend using hyphens).", "success": "That domain is available!", "error": "Error finding domain availability.", diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index b659b117e..4ef4efbba 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -115,7 +115,7 @@ function inlineToast(el, id, style, msg) { toast.className = `usa-alert usa-alert--${style} usa-alert--slim`; toastBody.classList.add("usa-alert__body"); p.classList.add("usa-alert__text"); - p.innerText = msg; + p.innerHTML = msg; toastBody.appendChild(p); toast.appendChild(toastBody); el.parentNode.insertBefore(toast, el.nextSibling); From 614da492dbb9f98cfe8121b710b21afc5971fc77 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 25 Nov 2023 07:45:46 -0500 Subject: [PATCH 096/119] added form level checking for duplicate entries in nameserver form --- src/registrar/forms/domain.py | 26 ++++++++++++++++++++++++++ src/registrar/utility/errors.py | 7 +++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index ae83650cb..965880354 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -117,8 +117,34 @@ class DomainNameserverForm(forms.Form): self.add_error("ip", str(e)) +class BaseNameserverFormset(forms.BaseFormSet): + def clean(self): + """ + Check for duplicate entries in the formset. + """ + if any(self.errors): + # Don't bother validating the formset unless each form is valid on its own + return + + data = [] + duplicates = [] + + for form in self.forms: + if form.cleaned_data: + value = form.cleaned_data['server'] + if value in data: + form.add_error( + "server", + NameserverError(code=nsErrorCodes.DUPLICATE_HOST, nameserver=value), + ) + duplicates.append(value) + else: + data.append(value) + + NameserverFormset = formset_factory( DomainNameserverForm, + formset=BaseNameserverFormset, extra=1, max_num=13, validate_max=True, diff --git a/src/registrar/utility/errors.py b/src/registrar/utility/errors.py index 420c616cb..52b1ea1d3 100644 --- a/src/registrar/utility/errors.py +++ b/src/registrar/utility/errors.py @@ -68,7 +68,8 @@ class NameserverErrorCodes(IntEnum): - 4 TOO_MANY_HOSTS more than the max allowed host values - 5 MISSING_HOST host is missing for a nameserver - 6 INVALID_HOST host is invalid for a nameserver - - 7 BAD_DATA bad data input for nameserver + - 7 DUPLICATE_HOST host is a duplicate + - 8 BAD_DATA bad data input for nameserver """ MISSING_IP = 1 @@ -77,7 +78,8 @@ class NameserverErrorCodes(IntEnum): TOO_MANY_HOSTS = 4 MISSING_HOST = 5 INVALID_HOST = 6 - BAD_DATA = 7 + DUPLICATE_HOST = 7 + BAD_DATA = 8 class NameserverError(Exception): @@ -93,6 +95,7 @@ class NameserverError(Exception): NameserverErrorCodes.TOO_MANY_HOSTS: ("Too many hosts provided, you may not have more than 13 nameservers."), NameserverErrorCodes.MISSING_HOST: ("Name server must be provided to enter IP address."), NameserverErrorCodes.INVALID_HOST: ("Enter a name server in the required format, like ns1.example.com"), + NameserverErrorCodes.DUPLICATE_HOST: ("Remove duplicate entry"), NameserverErrorCodes.BAD_DATA: ( "There’s something wrong with the name server information you provided. " "If you need help email us at help@get.gov." From 952cc6f46bb9e45932590a764fd7c2ea718b71cb Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 25 Nov 2023 08:09:33 -0500 Subject: [PATCH 097/119] added test case --- src/registrar/tests/test_views.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 936c344f7..9f423e276 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1462,6 +1462,30 @@ class TestDomainNameservers(TestDomainOverview): status_code=200, ) + def test_domain_nameservers_form_submit_duplicate_host(self): + """Nameserver form catches error when host is duplicated. + + Uses self.app WebTest because we need to interact with forms. + """ + # initial nameservers page has one server with two ips + nameservers_page = self.app.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # attempt to submit the form with duplicate host names of fake.host.com + nameservers_page.form["form-0-ip"] = "" + nameservers_page.form["form-1-server"] = "fake.host.com" + with less_console_noise(): # swallow log warning message + result = nameservers_page.form.submit() + # form submission was a post with an error, response should be a 200 + # error text appears twice, once at the top of the page, once around + # the required field. remove duplicate entry + self.assertContains( + result, + str(NameserverError(code=NameserverErrorCodes.DUPLICATE_HOST)), + count=2, + status_code=200, + ) + def test_domain_nameservers_form_submit_glue_record_not_allowed(self): """Nameserver form catches error when IP is present but host not subdomain. From b56b95ba086316c9c3a30e19a073f9e4e14810ba Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 25 Nov 2023 08:11:42 -0500 Subject: [PATCH 098/119] formatted for linter --- src/registrar/forms/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index 965880354..b8efdae49 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -131,7 +131,7 @@ class BaseNameserverFormset(forms.BaseFormSet): for form in self.forms: if form.cleaned_data: - value = form.cleaned_data['server'] + value = form.cleaned_data["server"] if value in data: form.add_error( "server", From 1f4fcf1225a5aa605b1c2b1a6d256dc3b94a0570 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 25 Nov 2023 10:04:24 -0500 Subject: [PATCH 099/119] fixed whitespace bug in ips in nameserver; created test case --- src/registrar/forms/domain.py | 1 + src/registrar/tests/common.py | 32 ++++++++++++++++++++------- src/registrar/tests/test_views.py | 36 +++++++++++++++++++++++++++++-- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/src/registrar/forms/domain.py b/src/registrar/forms/domain.py index ae83650cb..9c09467cd 100644 --- a/src/registrar/forms/domain.py +++ b/src/registrar/forms/domain.py @@ -67,6 +67,7 @@ class DomainNameserverForm(forms.Form): ip = cleaned_data.get("ip", None) # remove ANY spaces in the ip field ip = ip.replace(" ", "") + cleaned_data["ip"] = ip domain = cleaned_data.get("domain", "") ip_list = self.extract_ip_list(ip) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 9a062106f..8a971474d 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -859,15 +859,9 @@ class MockEppLib(TestCase): case commands.UpdateDomain: return self.mockUpdateDomainCommands(_request, cleaned) case commands.CreateHost: - return MagicMock( - res_data=[self.mockDataHostChange], - code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, - ) + return self.mockCreateHostCommands(_request, cleaned) case commands.UpdateHost: - return MagicMock( - res_data=[self.mockDataHostChange], - code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, - ) + return self.mockUpdateHostCommands(_request, cleaned) case commands.DeleteHost: return MagicMock( res_data=[self.mockDataHostChange], @@ -882,6 +876,28 @@ class MockEppLib(TestCase): case _: return MagicMock(res_data=[self.mockDataInfoHosts]) + def mockCreateHostCommands(self, _request, cleaned): + test_ws_ip = common.Ip(addr="1.1. 1.1") + addrs_submitted = getattr(_request, "addrs", []) + if test_ws_ip in addrs_submitted: + raise RegistryError(code=ErrorCode.PARAMETER_VALUE_RANGE_ERROR) + else: + return MagicMock( + res_data=[self.mockDataHostChange], + code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, + ) + + def mockUpdateHostCommands(self, _request, cleaned): + test_ws_ip = common.Ip(addr="1.1. 1.1") + addrs_submitted = getattr(_request, "addrs", []) + if test_ws_ip in addrs_submitted: + raise RegistryError(code=ErrorCode.PARAMETER_VALUE_RANGE_ERROR) + else: + return MagicMock( + res_data=[self.mockDataHostChange], + code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, + ) + def mockUpdateDomainCommands(self, _request, cleaned): if getattr(_request, "name", None) == "dnssec-invalid.gov": raise RegistryError(code=ErrorCode.PARAMETER_VALUE_RANGE_ERROR) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 936c344f7..39b23b546 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1462,6 +1462,38 @@ class TestDomainNameservers(TestDomainOverview): status_code=200, ) + def test_domain_nameservers_form_submit_whitespace(self): + """Nameserver form removes whitespace from ip. + + Uses self.app WebTest because we need to interact with forms. + """ + nameserver1 = "ns1.igorville.gov" + nameserver2 = "ns2.igorville.gov" + valid_ip = "1.1. 1.1" + # initial nameservers page has one server with two ips + # have to throw an error in order to test that the whitespace has been stripped from ip + nameservers_page = self.app.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain.id})) + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + # attempt to submit the form without one host and an ip with whitespace + nameservers_page.form["form-0-server"] = nameserver1 + nameservers_page.form["form-1-ip"] = valid_ip + nameservers_page.form["form-1-server"] = nameserver2 + with less_console_noise(): # swallow log warning message + result = nameservers_page.form.submit() + # form submission was a post with an ip address which has been stripped of whitespace, + # response should be a 302 to success page + self.assertEqual(result.status_code, 302) + self.assertEqual( + result["Location"], + reverse("domain-dns-nameservers", kwargs={"pk": self.domain.id}), + ) + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + page = result.follow() + # in the event of a generic nameserver error from registry error, there will be a 302 + # with an error message displayed, so need to follow 302 and test for success message + self.assertContains(page, "The name servers for this domain have been updated") + def test_domain_nameservers_form_submit_glue_record_not_allowed(self): """Nameserver form catches error when IP is present but host not subdomain. @@ -1553,7 +1585,7 @@ class TestDomainNameservers(TestDomainOverview): """ nameserver1 = "ns1.igorville.gov" nameserver2 = "ns2.igorville.gov" - invalid_ip = "127.0.0.1" + valid_ip = "127.0.0.1" # initial nameservers page has one server with two ips nameservers_page = self.app.get(reverse("domain-dns-nameservers", kwargs={"pk": self.domain.id})) session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] @@ -1562,7 +1594,7 @@ class TestDomainNameservers(TestDomainOverview): # only one has ips nameservers_page.form["form-0-server"] = nameserver1 nameservers_page.form["form-1-server"] = nameserver2 - nameservers_page.form["form-1-ip"] = invalid_ip + nameservers_page.form["form-1-ip"] = valid_ip with less_console_noise(): # swallow log warning message result = nameservers_page.form.submit() # form submission was a successful post, response should be a 302 From 22a21b5bf4a9af428c3349de94ec8cc8674c9b81 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Sat, 25 Nov 2023 10:22:34 -0500 Subject: [PATCH 100/119] format for linting --- src/registrar/tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index 8a971474d..d745669e5 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -897,7 +897,7 @@ class MockEppLib(TestCase): res_data=[self.mockDataHostChange], code=ErrorCode.COMMAND_COMPLETED_SUCCESSFULLY, ) - + def mockUpdateDomainCommands(self, _request, cleaned): if getattr(_request, "name", None) == "dnssec-invalid.gov": raise RegistryError(code=ErrorCode.PARAMETER_VALUE_RANGE_ERROR) From e314a5ff77fd728f64b29791a666a2105fd3cca8 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 27 Nov 2023 08:02:58 -0700 Subject: [PATCH 101/119] Update tests + linter --- src/registrar/admin.py | 3 ++- src/registrar/tests/test_admin.py | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/registrar/admin.py b/src/registrar/admin.py index ff0f71426..6585d602a 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -343,10 +343,11 @@ class UserDomainRoleAdmin(ListHeaderAdmin): class DomainInvitationAdmin(ListHeaderAdmin): """Custom domain invitation admin class.""" + class Meta: model = models.DomainInvitation fields = "__all__" - + _meta = Meta() # Columns diff --git a/src/registrar/tests/test_admin.py b/src/registrar/tests/test_admin.py index 089e569eb..bff27a83d 100644 --- a/src/registrar/tests/test_admin.py +++ b/src/registrar/tests/test_admin.py @@ -849,26 +849,25 @@ class TestDomainApplicationAdmin(MockEppLib): class DomainInvitationAdminTest(TestCase): + """Tests for the DomainInvitation page""" + def setUp(self): + """Create a client object""" + self.client = Client(HTTP_HOST="localhost:8080") self.factory = RequestFactory() self.admin = ListHeaderAdmin(model=DomainInvitationAdmin, admin_site=AdminSite()) - self.client = Client(HTTP_HOST="localhost:8080") self.superuser = create_superuser() - + def tearDown(self): - # delete any applications too + """Delete all DomainInvitation objects""" DomainInvitation.objects.all().delete() - DomainApplication.objects.all().delete() - User.objects.all().delete() def test_get_filters(self): + """Ensures that our filters are displaying correctly""" # Have to get creative to get past linter p = "adminpass" self.client.login(username="superuser", password=p) - # Mock a user - user = mock_user() - response = self.client.get( "/admin/registrar/domaininvitation/", {}, @@ -879,6 +878,13 @@ class DomainInvitationAdminTest(TestCase): self.assertContains(response, "invited", count=4) self.assertContains(response, "retrieved", count=4) + # Check for the HTML context specificially + invited_html = 'invited' + retrieved_html = 'retrieved' + + self.assertContains(response, invited_html, count=1) + self.assertContains(response, retrieved_html, count=1) + class ListHeaderAdminTest(TestCase): def setUp(self): From 6ace0f2862ef8862c03851971c04f440159bee12 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 27 Nov 2023 13:02:26 -0700 Subject: [PATCH 102/119] Update test_reports.py --- src/registrar/tests/test_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 14573ab65..52b971601 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -50,6 +50,7 @@ class ExportDataTest(TestCase): ) def tearDown(self): + # Dummy push - will remove Domain.objects.all().delete() DomainInformation.objects.all().delete() User.objects.all().delete() From 315638c020742c17d92c4933477a81bb65e0bd9e Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 15:08:42 -0500 Subject: [PATCH 103/119] updated comments for code legibility --- src/registrar/views/domain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index a9d0d4510..3aac32531 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -154,13 +154,13 @@ class DomainView(DomainBaseView): def in_editable_state(self, pk): """Override in_editable_state from DomainPermission - Allow detail page to be editable""" + Allow detail page to be viewable""" requested_domain = None if Domain.objects.filter(id=pk).exists(): requested_domain = Domain.objects.get(id=pk) - # if domain is editable return true + # return true if the domain exists, this will allow the detail page to load if requested_domain: return True return False From abe35b9d633b7a2952e148a6cb6ad842539409b7 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 18:02:43 -0500 Subject: [PATCH 104/119] changed the order of the permissions checking as the manage domain check was firing before editable check and allowing access --- src/registrar/views/utility/mixins.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/registrar/views/utility/mixins.py b/src/registrar/views/utility/mixins.py index 596873cf3..aaa2e849c 100644 --- a/src/registrar/views/utility/mixins.py +++ b/src/registrar/views/utility/mixins.py @@ -46,6 +46,10 @@ class DomainPermission(PermissionsLoginMixin): if pk is None: raise ValueError("Primary key is None") + # test if domain in editable state + if not self.in_editable_state(pk): + return False + if self.can_access_other_user_domains(pk): return True @@ -53,10 +57,6 @@ class DomainPermission(PermissionsLoginMixin): if not UserDomainRole.objects.filter(user=self.request.user, domain__id=pk).exists(): return False - # test if domain in editable state - if not self.in_editable_state(pk): - return False - # if we need to check more about the nature of role, do it here. return True From 20c13a5d2f97dfc8a43cf3d95e29d4fd8f94bf5f Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 19:16:45 -0500 Subject: [PATCH 105/119] when in read only mode remove add dns name server button and text --- src/registrar/templates/domain_detail.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/registrar/templates/domain_detail.html b/src/registrar/templates/domain_detail.html index 2bd6aa0ed..8d471d57e 100644 --- a/src/registrar/templates/domain_detail.html +++ b/src/registrar/templates/domain_detail.html @@ -31,9 +31,11 @@ {% if domain.nameservers|length > 0 %} {% include "includes/summary_item.html" with title='DNS name servers' domains='true' value=domain.nameservers list='true' edit_link=url editable=domain.is_editable %} {% else %} + {% if domain.is_editable %}

    DNS name servers

    No DNS name servers have been added yet. Before your domain can be used we’ll need information about your domain name servers.

    - Add DNS name servers + Add DNS name servers + {% endif %} {% endif %} {% url 'domain-org-name-address' pk=domain.id as url %} From 3ce76de1ff61b370f6e24e145cc413b39abcfa80 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 19:29:00 -0500 Subject: [PATCH 106/119] when in read only mode and no dns name servers, add empty section for dns name servers instead of button to add --- src/registrar/templates/domain_detail.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/registrar/templates/domain_detail.html b/src/registrar/templates/domain_detail.html index 8d471d57e..81a350f82 100644 --- a/src/registrar/templates/domain_detail.html +++ b/src/registrar/templates/domain_detail.html @@ -35,6 +35,8 @@

    DNS name servers

    No DNS name servers have been added yet. Before your domain can be used we’ll need information about your domain name servers.

    Add DNS name servers + {% else %} + {% include "includes/summary_item.html" with title='DNS name servers' domains='true' value='' edit_link=url editable=domain.is_editable %} {% endif %} {% endif %} From 128efb6a54b2a489268cab555720dd213a3716ef Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 20:03:38 -0500 Subject: [PATCH 107/119] updated comment --- src/registrar/models/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 1e4ab0e17..94430fb36 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -212,7 +212,7 @@ class Domain(TimeStampedModel, DomainHelper): @Cache def registry_expiration_date(self) -> date: """Get or set the `ex_date` element from the registry. - Additionally, update the expiration date in the registrar""" + Additionally, _get_property updates the expiration date in the registrar""" try: return self._get_property("ex_date") except Exception as e: From c2cd8ce5b656361265d534c84bbf77c1af7d038a Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 20:30:21 -0500 Subject: [PATCH 108/119] updated test for debugging --- src/registrar/tests/test_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 8493a3e75..6922b1b9a 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1222,8 +1222,10 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): def test_domain_detail_link_works(self): home_page = self.app.get("/") self.assertContains(home_page, "igorville.gov") + print(home_page) # click the "Edit" link detail_page = home_page.click("Manage", index=0) + print(detail_page) self.assertContains(detail_page, "igorville.gov") self.assertContains(detail_page, "Status") From 3ddc9260f2bd2c286175ca67c6e1df3f46071b0b Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 20:38:11 -0500 Subject: [PATCH 109/119] updated test for debugging --- src/registrar/tests/test_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 6922b1b9a..903027988 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1219,6 +1219,8 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): self.app.set_user(self.user.username) self.client.force_login(self.user) + +class TestDomainDetail(TestDomainOverview): def test_domain_detail_link_works(self): home_page = self.app.get("/") self.assertContains(home_page, "igorville.gov") From 2a31c5410baa62900d7e6de3fa6c12319299e12c Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 20:44:34 -0500 Subject: [PATCH 110/119] updated test for debugging --- src/registrar/tests/test_views.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 903027988..13b219f20 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1231,7 +1231,7 @@ class TestDomainDetail(TestDomainOverview): self.assertContains(detail_page, "igorville.gov") self.assertContains(detail_page, "Status") - def test_domain_overview_blocked_for_ineligible_user(self): + def test_domain_detail_blocked_for_ineligible_user(self): """We could easily duplicate this test for all domain management views, but a single url test should be solid enough since all domain management pages share the same permissions class""" @@ -1243,7 +1243,7 @@ class TestDomainDetail(TestDomainOverview): response = self.client.get(reverse("domain", kwargs={"pk": self.domain.id})) self.assertEqual(response.status_code, 403) - def test_domain_overview_allowed_for_on_hold(self): + def test_domain_detail_allowed_for_on_hold(self): """Test that the domain overview page displays for on hold domain""" home_page = self.app.get("/") self.assertContains(home_page, "on-hold.gov") @@ -1252,7 +1252,7 @@ class TestDomainDetail(TestDomainOverview): detail_page = self.client.get(reverse("domain", kwargs={"pk": self.domain_on_hold.id})) self.assertNotContains(detail_page, "Edit") - def test_domain_see_just_nameserver(self): + def test_domain_detail_see_just_nameserver(self): home_page = self.app.get("/") self.assertContains(home_page, "justnameserver.com") @@ -1263,7 +1263,7 @@ class TestDomainDetail(TestDomainOverview): self.assertContains(detail_page, "ns1.justnameserver.com") self.assertContains(detail_page, "ns2.justnameserver.com") - def test_domain_see_nameserver_and_ip(self): + def test_domain_detail_see_nameserver_and_ip(self): home_page = self.app.get("/") self.assertContains(home_page, "nameserverwithip.gov") @@ -1279,7 +1279,7 @@ class TestDomainDetail(TestDomainOverview): self.assertContains(detail_page, "(1.2.3.4,") self.assertContains(detail_page, "2.3.4.5)") - def test_domain_with_no_information_or_application(self): + def test_domain_detail_with_no_information_or_application(self): """Test that domain management page returns 200 and displays error when no domain information or domain application exist""" # have to use staff user for this test From 4a3d261de1bd1cc3fba5ef75d83176f219a5b883 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 20:47:51 -0500 Subject: [PATCH 111/119] removed redundant tests; removed debugging from test --- src/registrar/tests/test_views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index 13b219f20..88771ebab 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1224,10 +1224,8 @@ class TestDomainDetail(TestDomainOverview): def test_domain_detail_link_works(self): home_page = self.app.get("/") self.assertContains(home_page, "igorville.gov") - print(home_page) # click the "Edit" link detail_page = home_page.click("Manage", index=0) - print(detail_page) self.assertContains(detail_page, "igorville.gov") self.assertContains(detail_page, "Status") From 29962570a7b2936a3f420b2b82b763c14f1e8820 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Mon, 27 Nov 2023 21:02:31 -0500 Subject: [PATCH 112/119] fixed redundancy in test cases --- src/registrar/tests/test_views.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index d6bad9cdf..d2fdbc14f 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1219,6 +1219,8 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): self.app.set_user(self.user.username) self.client.force_login(self.user) + +class TestDomainDetail(TestDomainOverview): def test_domain_detail_link_works(self): home_page = self.app.get("/") self.assertContains(home_page, "igorville.gov") @@ -1227,7 +1229,7 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): self.assertContains(detail_page, "igorville.gov") self.assertContains(detail_page, "Status") - def test_domain_overview_blocked_for_ineligible_user(self): + def test_domain_detail_blocked_for_ineligible_user(self): """We could easily duplicate this test for all domain management views, but a single url test should be solid enough since all domain management pages share the same permissions class""" @@ -1239,7 +1241,7 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): response = self.client.get(reverse("domain", kwargs={"pk": self.domain.id})) self.assertEqual(response.status_code, 403) - def test_domain_overview_allowed_for_on_hold(self): + def test_domain_detail_allowed_for_on_hold(self): """Test that the domain overview page displays for on hold domain""" home_page = self.app.get("/") self.assertContains(home_page, "on-hold.gov") @@ -1248,7 +1250,7 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): detail_page = self.client.get(reverse("domain", kwargs={"pk": self.domain_on_hold.id})) self.assertNotContains(detail_page, "Edit") - def test_domain_see_just_nameserver(self): + def test_domain_detail_see_just_nameserver(self): home_page = self.app.get("/") self.assertContains(home_page, "justnameserver.com") @@ -1259,7 +1261,7 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): self.assertContains(detail_page, "ns1.justnameserver.com") self.assertContains(detail_page, "ns2.justnameserver.com") - def test_domain_see_nameserver_and_ip(self): + def test_domain_detail_see_nameserver_and_ip(self): home_page = self.app.get("/") self.assertContains(home_page, "nameserverwithip.gov") @@ -1275,7 +1277,7 @@ class TestDomainOverview(TestWithDomainPermissions, WebTest): self.assertContains(detail_page, "(1.2.3.4,") self.assertContains(detail_page, "2.3.4.5)") - def test_domain_with_no_information_or_application(self): + def test_domain_detail_with_no_information_or_application(self): """Test that domain management page returns 200 and displays error when no domain information or domain application exist""" # have to use staff user for this test From 7923c27fc24b07f79dec9895fb3679527578453e Mon Sep 17 00:00:00 2001 From: Michelle Rago <60157596+michelle-rago@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:14:47 -0500 Subject: [PATCH 113/119] Change "Add another user" to "Add a domain manager" (#1386) * Change "Add another user" to "Add a domain manager" * Change "Add another user" to "Add a domain manager" * Update test_views.py to change "Add another user" to "Add a domain manager" --- src/registrar/templates/domain_add_user.html | 4 ++-- src/registrar/templates/domain_users.html | 2 +- src/registrar/tests/test_views.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/registrar/templates/domain_add_user.html b/src/registrar/templates/domain_add_user.html index 87b141570..d8cae576a 100644 --- a/src/registrar/templates/domain_add_user.html +++ b/src/registrar/templates/domain_add_user.html @@ -4,10 +4,10 @@ {% block title %}Add another user{% endblock %} {% block domain_content %} -

    Add another user

    +

    Add a domain manager

    You can add another user to help manage your domain. They will need to sign - into the .gov registrar with their Login.gov account. + in to the .gov registrar with their Login.gov account.

    diff --git a/src/registrar/templates/domain_users.html b/src/registrar/templates/domain_users.html index e1ea0b601..8ee837708 100644 --- a/src/registrar/templates/domain_users.html +++ b/src/registrar/templates/domain_users.html @@ -53,7 +53,7 @@ Add another user + Add a domain manager diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index d6bad9cdf..3b0287add 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -1305,12 +1305,12 @@ class TestDomainManagers(TestDomainOverview): def test_domain_managers_add_link(self): """Button to get to user add page works.""" management_page = self.app.get(reverse("domain-users", kwargs={"pk": self.domain.id})) - add_page = management_page.click("Add another user") - self.assertContains(add_page, "Add another user") + add_page = management_page.click("Add a domain manager") + self.assertContains(add_page, "Add a domain manager") def test_domain_user_add(self): response = self.client.get(reverse("domain-users-add", kwargs={"pk": self.domain.id})) - self.assertContains(response, "Add another user") + self.assertContains(response, "Add a domain manager") def test_domain_user_add_form(self): """Adding an existing user works.""" From 80880a4ca1d8f6cc9a440ffcb73c90d57a0e70d8 Mon Sep 17 00:00:00 2001 From: Michelle Rago <60157596+michelle-rago@users.noreply.github.com> Date: Tue, 28 Nov 2023 11:00:33 -0500 Subject: [PATCH 114/119] Update success message for org name and mailing address (#1387) Update domain.py --- src/registrar/views/domain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/views/domain.py b/src/registrar/views/domain.py index 3aac32531..4d91ddc66 100644 --- a/src/registrar/views/domain.py +++ b/src/registrar/views/domain.py @@ -197,7 +197,7 @@ class DomainOrgNameAddressView(DomainFormBaseView): """The form is valid, save the organization name and mailing address.""" form.save() - messages.success(self.request, "The organization name and mailing address has been updated.") + messages.success(self.request, "The organization information has been updated.") # superclass has the redirect return super().form_valid(form) From e24bf1ae58ab935211db2e4e4f0004fcc745b805 Mon Sep 17 00:00:00 2001 From: Michelle Rago <60157596+michelle-rago@users.noreply.github.com> Date: Tue, 28 Nov 2023 11:00:58 -0500 Subject: [PATCH 115/119] Change "Active users" to "Domain managers" on domain managers page (#1407) --- src/registrar/templates/domain_users.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/templates/domain_users.html b/src/registrar/templates/domain_users.html index 8ee837708..0eecd35b3 100644 --- a/src/registrar/templates/domain_users.html +++ b/src/registrar/templates/domain_users.html @@ -25,8 +25,8 @@ {% if domain.permissions %}
    -

    Active users

    - +

    Domain managers

    + From bd6eabe9165f334183a1881b4e7f473365f87458 Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 28 Nov 2023 12:17:47 -0500 Subject: [PATCH 116/119] Typo fixes in ops readme and rotate secrets --- docs/operations/README.md | 4 ++-- docs/operations/runbooks/rotate_application_secrets.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/operations/README.md b/docs/operations/README.md index 4c7f182bd..0629608dd 100644 --- a/docs/operations/README.md +++ b/docs/operations/README.md @@ -55,9 +55,9 @@ In the case where a bug fix or feature needs to be added outside of the normal c 1. Code will need to be branched NOT off of main, but off of the same commit as the most recent stable commit. This should be the one tagged with the most recent vX.XX.XX value. 2. After making the bug fix, the approved PR branch will not be merged yet, instead it will be tagged with a new release tag, incrementing the patch value from the last commit number. -3. If main and stable are on the the same commit then merge this branch into the staging using the staging release tag (staging-). +3. If main and stable are on the the same commit then merge this branch into staging using the staging release tag (staging-). 4. If staging is already ahead stable, you may need to create another branch that is based off of the current staging commit, merge in your code change and then tag that branch with the staging release. -5. Wait to merge your original branch until both deploys finish. Once they succeed then merge to main per the usual process. +5. Wait to merge your original branch until both deploys finish. Once they succeed then merge to main per the usual process. ## Serving static assets We are using [WhiteNoise](http://whitenoise.evans.io/en/stable/index.html) plugin to serve our static assets on cloud.gov. This plugin is added to the `MIDDLEWARE` list in our apps `settings.py`. diff --git a/docs/operations/runbooks/rotate_application_secrets.md b/docs/operations/runbooks/rotate_application_secrets.md index 78c402efe..a776e60b8 100644 --- a/docs/operations/runbooks/rotate_application_secrets.md +++ b/docs/operations/runbooks/rotate_application_secrets.md @@ -112,7 +112,7 @@ base64 -i client.key base64 -i client.crt ``` -You'll need to give the new certificate to the registry vendor _before_ rotating it in production. Once it has been accepted by the vender, make sure to update the kdbx file on Google Drive. +You'll need to give the new certificate to the registry vendor _before_ rotating it in production. Once it has been accepted by the vendor, make sure to update the kdbx file on Google Drive. ## REGISTRY_HOSTNAME From b6b251b287bd6b03c393ea428f0aaa697698026b Mon Sep 17 00:00:00 2001 From: Rachid Mrad Date: Tue, 28 Nov 2023 13:32:48 -0500 Subject: [PATCH 117/119] more minor typo fixes --- .../decisions/0024-production-release-cadence.md | 6 +++--- docs/django-admin/roles.md | 6 +++--- docs/operations/runbooks/update_python_dependencies.md | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/architecture/decisions/0024-production-release-cadence.md b/docs/architecture/decisions/0024-production-release-cadence.md index 1020d3506..85bfcdfe2 100644 --- a/docs/architecture/decisions/0024-production-release-cadence.md +++ b/docs/architecture/decisions/0024-production-release-cadence.md @@ -15,9 +15,9 @@ Going into our first production launch we need a plan describing what our releas **Option 1:** Releasing to stable/staging once a sprint Releasing once a sprint would mean that we release the past sprint's work to stable at the end of the current sprint. At the same point, the current sprint's work would be pushed to staging, thus making staging a full sprint ahead of stable. While this is more straight forward, it means our users would have to wait longer to see changes that weren't deemed critical. **Option 2:** Releasing to stable/staging once a week -Releasing once a week would follow the same flow but with code being released to staging one week before the same code is released to stable. This would make stable only one week behind staging and would allow us to roll out minor bug fixes and faster with greater speed. The negative side is that we have less time to see if errors occur on staging +Releasing once a week would follow the same flow but with code being released to staging one week before the same code is released to stable. This would make stable only one week behind staging and would allow us to roll out minor bug fixes faster. The negative side is that we have less time to see if errors occur on staging. -In both of the above scenarios the release date would fall on the same day of the week that the sprint starts, which is currently a Wednesday. Additionally, in both scenarios the release commits would eventually be tagged with both a staging and stable tag. Furthermore, critical bugs or features would be exempt from these restrictions based on the product owner's discretion. +In both of the above scenarios, the release date would fall on the same day of the week that the sprint starts which is currently a Wednesday. Additionally, in both scenarios the release commits would eventually be tagged with both a staging and stable tag. Furthermore, critical bugs or features would be exempt from these restrictions based on the product owner's discretion. ## Decision @@ -25,6 +25,6 @@ We decided to go with option 2 and release once a week once in production. This ## Consequences -Work not completed by end of the sprint will have to wait to be added to stable. Also, making quick fixes for bugs that are found on stable will be a little more complicated to fix. +Work not completed by end of the sprint will have to wait to be added to stable. Also, making quick fixes for bugs that are found on stable will be a little more complicated. When first going into production, staging and stable will start with the same code base. The following week a new release will be made to staging, but not stable as no code will have been on staging long enough to warrant another release. Thus just at the start of launch stable will be essentially frozen for 2 weeks, not one. diff --git a/docs/django-admin/roles.md b/docs/django-admin/roles.md index 6fc0d385e..c527bbfa5 100644 --- a/docs/django-admin/roles.md +++ b/docs/django-admin/roles.md @@ -19,7 +19,7 @@ To do this, do the following: 3. Click on their username, then scroll down to the `User Permissions` section. 4. Under `User Permissions`, see the `Groups` table which has a column for `Available groups` and `Chosen groups`. Select the permission you want from the `Available groups` column and click the right arrow to move it to the `Chosen groups`. Note, if you want this user to be an analyst select `cisa_analysts_group`, otherwise select the `full_access_group`. 5. (Optional) If the user needs access to django admin (such as an analyst), then you will also need to make sure "Staff Status" is checked. This can be found in the same `User Permissions` section right below the checkbox for `Active`. -6. Click `Save` to apply all changes +6. Click `Save` to apply all changes. ## Removing a user group permission via django-admin @@ -30,7 +30,7 @@ If an employee was given the wrong permissions or has had a change in roles that 3. In this table, select the permission you want to remove from the `Chosen groups` and then click the left facing arrow to move the permission to `Available groups`. 4. Depending on the scenario you may now need to add the opposite permission group to the `Chosen groups` section, please see the section above for instructions on how to do that. 5. If the user should no longer see the admin page, you must ensure that under `User Permissions`, `Staff status` is NOT checked. -6. Click `Save` to apply all changes +6. Click `Save` to apply all changes. ## Editing group permissions through code @@ -40,4 +40,4 @@ We can edit and deploy new group permissions by: 2. Duplicating migration `0036_create_groups_01` and running migrations (append the name with a version number to help django detect the migration eg 0037_create_groups_02) -3. Making sure to update the dependency on the new migration with the previous migration \ No newline at end of file +3. Making sure to update the dependency on the new migration with the previous migration. \ No newline at end of file diff --git a/docs/operations/runbooks/update_python_dependencies.md b/docs/operations/runbooks/update_python_dependencies.md index 16475d3db..ea206bbde 100644 --- a/docs/operations/runbooks/update_python_dependencies.md +++ b/docs/operations/runbooks/update_python_dependencies.md @@ -3,7 +3,7 @@ 1. Check the [Pipfile](../../../src/Pipfile) for pinned dependencies and manually adjust the version numbers -1. Run +2. Run cd src docker-compose run app bash -c "pipenv lock && pipenv requirements > requirements.txt" @@ -14,6 +14,6 @@ The requirements.txt is used by Cloud.gov. It is needed to work around a bug in the CloudFoundry buildpack version of Pipenv that breaks on installing from a git repository. -1. (optional) Run `docker-compose stop` and `docker-compose build` to build a new image for local development with the updated dependencies. +3. (optional) Run `docker-compose stop` and `docker-compose build` to build a new image for local development with the updated dependencies. The reason for de-coupling the `build` and `lock` steps is to increase consistency between builds--a run of `build` will always get exactly the dependencies listed in `Pipfile.lock`, nothing more, nothing less. \ No newline at end of file From 89aa3534cf42192ee1cc249c91604031081f6e09 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 28 Nov 2023 17:10:20 -0500 Subject: [PATCH 118/119] fixed formatting of error message in javascript --- src/registrar/assets/js/get-gov.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/registrar/assets/js/get-gov.js b/src/registrar/assets/js/get-gov.js index 4ef4efbba..d069e8dc4 100644 --- a/src/registrar/assets/js/get-gov.js +++ b/src/registrar/assets/js/get-gov.js @@ -122,7 +122,7 @@ function inlineToast(el, id, style, msg) { } else { // update and show the existing message div toast.className = `usa-alert usa-alert--${style} usa-alert--slim`; - toast.querySelector("div p").innerText = msg; + toast.querySelector("div p").innerHTML = msg; makeVisible(toast); } } else { From f957c299b8e80ba8570535f54913880faa7eaf85 Mon Sep 17 00:00:00 2001 From: David Kennedy Date: Tue, 28 Nov 2023 17:12:12 -0500 Subject: [PATCH 119/119] added comment --- src/api/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/views.py b/src/api/views.py index 5e5365e58..a9f8d7692 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -21,6 +21,8 @@ DOMAIN_API_MESSAGES = { " For example, if you want www.city.gov, you would enter “city”" " (without the quotes).", "extra_dots": "Enter the .gov domain you want without any periods.", + # message below is considered safe; no user input can be inserted into the message + # body; public_site_url() function reads from local app settings and therefore safe "unavailable": mark_safe( # nosec "That domain isn’t available. " ""
    Domain usersDomain managers
    Email