zydronium/manage.get.gov
1
0
Fork 0
mirror of https://github.com/cisagov/manage.get.gov.git synced 2025-05-22 12:29:25 +02:00
Code Issues Projects Releases Packages Wiki Activity

Linting + minimize logging to a degree

Browse source
This commit is contained in:
zandercymatics 2023-11-08 17:37:59 -07:00
parent 7d798e0e43
commit 72d95f6fad
No known key found for this signature in database
GPG key ID: FF4636ABEC9682B7
11 changed files with 507 additions and 373 deletions
Download patch file Download diff file Expand all files Collapse all files

18
docs/operations/data_migration.md
View file

@ -240,7 +240,23 @@ This will allow Docker to mount the files to a container (under `/app`) for our
### STEP 1: Load Transition Domains
Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table.
##### Create a JSON file
In your chosen directory (either `src/tmp` or `src/migrationdata` depending on preference), create a json file called `migrationFilepaths.json`. This file will map to other urls
Example
```
{
"directory": "migrationdata/",
"agency_adhoc_filename": "20231009.agency.adhoc.dotgov.txt",
"authority_adhoc_filename": "authority.adhoc.dotgov.txt",
"contacts_filename": "escrow_contacts.daily.dotgov.GOV.txt",
"domain_adhoc_filename": "20231009.domaintypes.adhoc.dotgov.txt",
"domain_additional_filename": "20231009.domainadditionaldatalink.adhoc.dotgov.txt",
"domain_contacts_filename": "escrow_domain_contacts.daily.dotgov.GOV.txt",
"domain_escrow_filename": "escrow_domains.daily.dotgov.GOV.txt",
"domain_statuses_filename": "escrow_domain_statuses.daily.dotgov.GOV.txt",
"organization_adhoc_filename": "20231009.organization.adhoc.dotgov.txt"
}
```
##### LOCAL COMMAND
```shell
docker-compose exec app ./manage.py load_transition_domain migrationFilepaths.json --directory /app/tmp/ --debug --limitParse 10

109
src/registrar/management/commands/agency_data_extractor.py
View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
# Example command for running this script:
# docker compose run -T app ./manage.py agency_data_extractor 20231009.agency.adhoc.dotgov.txt --dir /app/tmp --debug
class Command(BaseCommand):
help = """Loads data for domains that are in transition
(populates transition_domain model objects)."""
@ -26,38 +27,40 @@ class Command(BaseCommand):
parser.add_argument(
"agency_data_filename", help="Data file with agency information"
)
parser.add_argument(
"--dir", default="migrationdata", help="Desired directory"
)
parser.add_argument("--dir", default="migrationdata", help="Desired directory")
parser.add_argument("--sep", default="|", help="Delimiter character")
parser.add_argument("--debug", help="Prints additional debug statements to the terminal", action=argparse.BooleanOptionalAction)
parser.add_argument(
"--debug",
help="Prints additional debug statements to the terminal",
action=argparse.BooleanOptionalAction,
)
parser.add_argument("--prompt", action=argparse.BooleanOptionalAction)
@staticmethod
def extract_agencies(
agency_data_filepath: str,
sep: str,
debug: bool
) -> [str]:
def extract_agencies(agency_data_filepath: str, sep: str, debug: bool) -> [str]:
"""Extracts all the agency names from the provided
agency file (skips any duplicates) and returns those
names in an array"""
agency_names = []
logger.info(f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}")
logger.info(
f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}"
)
with open(agency_data_filepath, "r") as agency_data_filepath: # noqa
for row in csv.reader(agency_data_filepath, delimiter=sep):
agency_name = row[1]
TerminalHelper.print_conditional(debug, f"Checking: {agency_name}")
if agency_name not in agency_names:
agency_names.append(agency_name)
logger.info(f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}")
logger.info(
f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}"
)
return agency_names
@staticmethod
def compare_agency_lists(provided_agencies: [str],
existing_agencies: [str],
debug: bool):
def compare_agency_lists(
provided_agencies: [str], existing_agencies: [str], debug: bool
):
"""
Compares new_agencies with existing_agencies and
provides the equivalent of an outer-join on the two
@ -69,27 +72,37 @@ class Command(BaseCommand):
for agency in provided_agencies:
if agency not in existing_agencies and agency not in new_agencies:
new_agencies.append(agency)
TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}")
TerminalHelper.print_conditional(
debug,
f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}",
)
possibly_unused_agencies = []
# 2 - Get all new agencies that we don't already have (We might want to ADD these to our list)
for agency in existing_agencies:
if agency not in provided_agencies and agency not in possibly_unused_agencies:
if (
agency not in provided_agencies
and agency not in possibly_unused_agencies
):
possibly_unused_agencies.append(agency)
TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}")
TerminalHelper.print_conditional(
debug,
f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}",
)
matched_agencies = []
for agency in provided_agencies:
if agency in existing_agencies:
matched_agencies.append(agency)
TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}")
TerminalHelper.print_conditional(
debug,
f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}",
)
# Print the summary of findings
# 1 - Print the list of agencies in the NEW list, which we do not already have
# 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies?
new_agencies_as_string = "{}".format(
",\n ".join(map(str, new_agencies))
)
new_agencies_as_string = "{}".format(",\n ".join(map(str, new_agencies)))
possibly_unused_agencies_as_string = "{}".format(
",\n ".join(map(str, possibly_unused_agencies))
)
@ -97,7 +110,8 @@ class Command(BaseCommand):
",\n ".join(map(str, matched_agencies))
)
logger.info(f"""
logger.info(
f"""
{TerminalColors.OKGREEN}
======================== SUMMARY OF FINDINGS ============================
{len(provided_agencies)} AGENCIES WERE PROVIDED in the agency file.
@ -117,13 +131,12 @@ class Command(BaseCommand):
These agencies are in our system, but not in the provided agency file:
{TerminalColors.YELLOW}{possibly_unused_agencies_as_string}
{TerminalColors.ENDC}
""")
"""
)
@staticmethod
def print_agency_list(agencies, filename):
full_agency_list_as_string = "{}".format(
",\n".join(map(str, agencies))
)
full_agency_list_as_string = "{}".format(",\n".join(map(str, agencies)))
logger.info(
f"\n{TerminalColors.YELLOW}"
f"\n{full_agency_list_as_string}"
@ -150,10 +163,13 @@ class Command(BaseCommand):
new_agencies = self.extract_agencies(agency_data_file, sep, debug)
hard_coded_agencies = DomainApplication.AGENCIES
transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency', flat=True).distinct()
transition_domain_agencies = (
TransitionDomain.objects.all()
.values_list("federal_agency", flat=True)
.distinct()
)
print(transition_domain_agencies)
merged_agencies = new_agencies
for agency in hard_coded_agencies:
if agency not in merged_agencies:
@ -168,19 +184,25 @@ class Command(BaseCommand):
# OPTION to compare the agency file to our hard-coded list
if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}")
prompt_successful = TerminalHelper.query_yes_no(
f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt:
self.compare_agency_lists(new_agencies, hard_coded_agencies, debug)
# OPTION to compare the agency file to Transition Domains
if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}")
prompt_successful = TerminalHelper.query_yes_no(
f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt:
self.compare_agency_lists(new_agencies, transition_domain_agencies, debug)
# OPTION to print out the full list of agencies from the agency file
if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}")
prompt_successful = TerminalHelper.query_yes_no(
f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt:
logger.info(
f"\n{TerminalColors.OKGREEN}"
@ -192,7 +214,9 @@ class Command(BaseCommand):
# OPTION to print out the full list of agencies from the agency file
if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}")
prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt:
logger.info(
f"\n{TerminalColors.OKGREEN}"
@ -204,7 +228,9 @@ class Command(BaseCommand):
# OPTION to print out the full list of agencies from the agency file
if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}")
prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt:
logger.info(
f"\n{TerminalColors.OKGREEN}"
@ -212,12 +238,15 @@ class Command(BaseCommand):
f"\nThese are all the agencies in the Transition Domains table."
f"\n\n{len(transition_domain_agencies)} TOTAL\n\n"
)
self.print_agency_list(transition_domain_agencies, "Transition_Domain_Agencies")
self.print_agency_list(
transition_domain_agencies, "Transition_Domain_Agencies"
)
# OPTION to print out the full list of agencies from the agency file
if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}")
prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt:
logger.info(
f"\n{TerminalColors.OKGREEN}"
@ -229,7 +258,9 @@ class Command(BaseCommand):
# OPTION to print out the full list of agencies from the agency file
if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}")
prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt:
logger.info(
f"\n{TerminalColors.OKGREEN}"
@ -237,4 +268,6 @@ class Command(BaseCommand):
f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file."
f"\n\n{len(merged_agencies)} TOTAL\n\n"
)
self.print_agency_list(merged_transition_agencies, "Merged_Transition_Domain_Agency_List")
self.print_agency_list(
merged_transition_agencies, "Merged_Transition_Domain_Agency_List"
)

27
src/registrar/management/commands/load_transition_domain.py
View file

@ -66,7 +66,7 @@ class Command(BaseCommand):
parser.add_argument(
"--infer_filenames",
action=argparse.BooleanOptionalAction,
help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting."
help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting.",
)
parser.add_argument(
@ -74,7 +74,7 @@ class Command(BaseCommand):
)
parser.add_argument(
"--domain_contacts_filename",
help="Data file with domain contact information"
help="Data file with domain contact information",
)
parser.add_argument(
"--contacts_filename",
@ -82,7 +82,7 @@ class Command(BaseCommand):
)
parser.add_argument(
"--domain_statuses_filename",
help="Data file with domain status information"
help="Data file with domain status information",
)
parser.add_argument(
"--agency_adhoc_filename",
@ -347,10 +347,12 @@ class Command(BaseCommand):
# If it does, update the options
options[key] = value
except Exception as err:
logger.error(f"""{TerminalColors.FAIL}There was an error loading the JSON responsible
logger.error(
f"""{TerminalColors.FAIL}There was an error loading the JSON responsible
for providing filepaths.
{TerminalColors.ENDC}
""")
"""
)
raise err
sep = args.sep
@ -370,11 +372,12 @@ class Command(BaseCommand):
## Variables for Additional TransitionDomain Information ##
# Main script filenames - these do not have defaults
domain_contacts_filename = None
try:
domain_contacts_filename = directory + options.get("domain_contacts_filename")
domain_contacts_filename = directory + options.get(
"domain_contacts_filename"
)
except TypeError as err:
logger.error(
f"Invalid filename of '{args.domain_contacts_filename}'"
@ -392,7 +395,9 @@ class Command(BaseCommand):
domain_statuses_filename = None
try:
domain_statuses_filename = directory + options.get("domain_statuses_filename")
domain_statuses_filename = directory + options.get(
"domain_statuses_filename"
)
except TypeError as err:
logger.error(
f"Invalid filename of '{args.domain_statuses_filename}'"
@ -468,7 +473,10 @@ class Command(BaseCommand):
new_entry_email = ""
new_entry_emailSent = False # set to False by default
TerminalHelper.print_conditional(True, f"Processing item {total_rows_parsed}: {new_entry_domain_name}")
TerminalHelper.print_conditional(
True,
f"Processing item {total_rows_parsed}: {new_entry_domain_name}",
)
# PART 1: Get the status
if new_entry_domain_name not in domain_status_dictionary:
@ -608,7 +616,6 @@ class Command(BaseCommand):
)
self.print_summary_status_findings(domains_without_status, outlier_statuses)
logger.info(
f"""{TerminalColors.OKGREEN}
============= FINISHED ===============

11
src/registrar/management/commands/master_domain_migrations.py
View file

@ -30,7 +30,6 @@ logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = """ """ # TODO: update this!
# ======================================================
# ================== ARGUMENTS ===================
# ======================================================
@ -129,7 +128,6 @@ class Command(BaseCommand):
action=argparse.BooleanOptionalAction,
)
# ======================================================
# =============== DATA ANALYSIS ==================
# ======================================================
@ -252,7 +250,6 @@ class Command(BaseCommand):
"""
)
# ======================================================
# ================= MIGRATIONS ===================
# ======================================================
@ -266,17 +263,13 @@ class Command(BaseCommand):
prompts_enabled: bool,
debug_max_entries_to_parse: int,
):
if file_directory and file_directory[-1] != "/":
file_directory += "/"
json_filepath = migration_json_filename
"""Runs the load_transition_domain script"""
# Create the command string
command_script = "load_transition_domain"
command_string = (
f"./manage.py {command_script} "
f"{json_filepath} "
)
command_string = f"./manage.py {command_script} " f"{json_filepath} "
if sep is not None and sep != "|":
command_string += f"--sep {sep} "
if reset_table:
@ -306,7 +299,7 @@ class Command(BaseCommand):
resetTable=reset_table,
debug=debug_on,
limitParse=debug_max_entries_to_parse,
directory=file_directory
directory=file_directory,
)
def run_transfer_script(self, debug_on: bool, prompts_enabled: bool):

190
src/registrar/management/commands/transfer_transition_domains_to_domains.py
View file

@ -39,7 +39,6 @@ class Command(BaseCommand):
help="Sets max number of entries to load, set to 0 to load all entries",
)
# ======================================================
# ===================== PRINTING ======================
# ======================================================
@ -67,12 +66,13 @@ class Command(BaseCommand):
""",
)
def parse_limit_reached(self,
debug_max_entries_to_parse: bool,
total_rows_parsed: int
def parse_limit_reached(
self, debug_max_entries_to_parse: bool, total_rows_parsed: int
) -> bool:
if (debug_max_entries_to_parse > 0
and total_rows_parsed >= debug_max_entries_to_parse):
if (
debug_max_entries_to_parse > 0
and total_rows_parsed >= debug_max_entries_to_parse
):
logger.info(
f"""{TerminalColors.YELLOW}
----PARSE LIMIT REACHED. HALTING PARSER.----
@ -156,13 +156,12 @@ class Command(BaseCommand):
""",
)
# ======================================================
# =================== DOMAIN =====================
# ======================================================
def update_or_create_domain(self,
transition_domain: TransitionDomain,
debug_on: bool) -> (Domain, bool):
def update_or_create_domain(
self, transition_domain: TransitionDomain, debug_on: bool
) -> (Domain, bool):
"""Given a transition domain, either finds & updates an existing
corresponding domain, or creates a new corresponding domain in
the Domain table.
@ -258,7 +257,6 @@ class Command(BaseCommand):
)
return (target_domain, True)
def update_domain_status(
self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool
) -> bool:
@ -289,7 +287,6 @@ class Command(BaseCommand):
return True
return False
# ======================================================
# ================ DOMAIN INVITATION ==================
# ======================================================
@ -336,22 +333,26 @@ class Command(BaseCommand):
# ======================================================
# ================ DOMAIN INFORMATION =================
# ======================================================
def update_domain_information(self, current: DomainInformation, target: DomainInformation, debug_on: bool) -> bool:
def update_domain_information(
self, current: DomainInformation, target: DomainInformation, debug_on: bool
) -> bool:
# DEBUG:
TerminalHelper.print_conditional(
debug_on,
(f"{TerminalColors.OKCYAN}"
(
f"{TerminalColors.OKCYAN}"
f"Updating: {current}"
f"{TerminalColors.ENDC}"), # noqa
f"{TerminalColors.ENDC}"
), # noqa
)
updated = False
fields_to_update = [
'organization_type',
'federal_type',
'federal_agency',
"organization_name"
"organization_type",
"federal_type",
"federal_agency",
"organization_name",
]
defaults = {field: getattr(target, field) for field in fields_to_update}
if current != target:
@ -364,10 +365,15 @@ class Command(BaseCommand):
def try_add_domain_information(self):
pass
def create_new_domain_info(self,
def create_new_domain_info(
self,
transition_domain: TransitionDomain,
domain: Domain) -> DomainInformation:
domain: Domain,
agency_choices,
fed_choices,
org_choices,
debug_on,
) -> DomainInformation:
org_type = transition_domain.organization_type
fed_type = transition_domain.federal_type
fed_agency = transition_domain.federal_agency
@ -388,32 +394,31 @@ class Command(BaseCommand):
case "Independent Intrastate":
org_type = ("special_district", "Special district")
valid_org_type = org_type in [(name, value) for name, value in DomainApplication.OrganizationChoices.choices]
valid_fed_type = fed_type in [value for name, value in DomainApplication.BranchChoices.choices]
valid_fed_agency = fed_agency in DomainApplication.AGENCIES
valid_org_type = org_type in org_choices
valid_fed_type = fed_type in fed_choices
valid_fed_agency = fed_agency in agency_choices
default_creator, _ = User.objects.get_or_create(username="System")
new_domain_info_data = {
'domain': domain,
'organization_name': transition_domain.organization_name,
"domain": domain,
"organization_name": transition_domain.organization_name,
"creator": default_creator,
}
if valid_org_type:
new_domain_info_data['organization_type'] = org_type[0]
else:
new_domain_info_data["organization_type"] = org_type[0]
elif debug_on:
logger.debug(f"No org type found on {domain.name}")
if valid_fed_type:
new_domain_info_data['federal_type'] = fed_type.lower()
pass
else:
new_domain_info_data["federal_type"] = fed_type.lower()
elif debug_on:
logger.debug(f"No federal type found on {domain.name}")
if valid_fed_agency:
new_domain_info_data['federal_agency'] = fed_agency
else:
new_domain_info_data["federal_agency"] = fed_agency
elif debug_on:
logger.debug(f"No federal agency found on {domain.name}")
new_domain_info = DomainInformation(**new_domain_info_data)
@ -421,16 +426,22 @@ class Command(BaseCommand):
# DEBUG:
TerminalHelper.print_conditional(
True,
(f"{TerminalColors.MAGENTA}"
(
f"{TerminalColors.MAGENTA}"
f"Created Domain Information template for: {new_domain_info}"
f"{TerminalColors.ENDC}"), # noqa
f"{TerminalColors.ENDC}"
), # noqa
)
return new_domain_info
def update_or_create_domain_information(self,
def update_or_create_domain_information(
self,
transition_domain: TransitionDomain,
debug_on: bool) -> (DomainInformation, bool):
agency_choices,
fed_choices,
org_choices,
debug_on: bool,
) -> (DomainInformation, bool):
transition_domain_name = transition_domain.domain_name
# Get associated domain
@ -444,20 +455,33 @@ class Command(BaseCommand):
)
return (None, None, False)
domain = domain_data.get()
template_domain_information = self.create_new_domain_info(transition_domain, domain)
template_domain_information = self.create_new_domain_info(
transition_domain,
domain,
agency_choices,
fed_choices,
org_choices,
debug_on,
)
target_domain_information = None
domain_information_exists = DomainInformation.objects.filter(domain__name=transition_domain_name).exists()
domain_information_exists = DomainInformation.objects.filter(
domain__name=transition_domain_name
).exists()
if domain_information_exists:
try:
# get the existing domain information object
target_domain_information = DomainInformation.objects.get(domain__name=transition_domain_name)
target_domain_information = DomainInformation.objects.get(
domain__name=transition_domain_name
)
# DEBUG:
TerminalHelper.print_conditional(
debug_on,
(f"{TerminalColors.FAIL}"
(
f"{TerminalColors.FAIL}"
f"Found existing entry in Domain Information table for:"
f"{transition_domain_name}"
f"{TerminalColors.ENDC}"), # noqa
f"{TerminalColors.ENDC}"
), # noqa
)
# for existing entry, update the status to
@ -493,15 +517,15 @@ class Command(BaseCommand):
# DEBUG:
TerminalHelper.print_conditional(
debug_on,
(f"{TerminalColors.OKCYAN}"
(
f"{TerminalColors.OKCYAN}"
f"Adding domain information for: "
f"{transition_domain_name}"
f"{TerminalColors.ENDC}"),
f"{TerminalColors.ENDC}"
),
)
return (target_domain_information, domain, True)
# ======================================================
# ===================== HANDLE ========================
# ======================================================
@ -536,7 +560,6 @@ class Command(BaseCommand):
# domain invitations to ADD
domain_invitations_to_create = []
# if we are limiting our parse (for testing purposes, keep
# track of total rows parsed)
total_rows_parsed = 0
@ -574,7 +597,9 @@ class Command(BaseCommand):
# ======================================================
# ====================== DOMAIN =======================
target_domain, was_created = self.update_or_create_domain(transition_domain, debug_on)
target_domain, was_created = self.update_or_create_domain(
transition_domain, debug_on
)
debug_string = ""
if target_domain is None:
@ -633,7 +658,6 @@ class Command(BaseCommand):
if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed):
break
# First, save all Domain objects to the database
Domain.objects.bulk_create(domains_to_create)
# DomainInvitation.objects.bulk_create(domain_invitations_to_create)
@ -643,6 +667,7 @@ class Command(BaseCommand):
# utilize.
# Then, create DomainInvitation objects
for invitation in domain_invitations_to_create:
if debug_on:
logger.info(f"Pairing invite to its domain...{invitation}")
existing_domain = Domain.objects.filter(name=invitation.domain.name)
# Make sure the related Domain object is saved
@ -650,9 +675,19 @@ class Command(BaseCommand):
invitation.domain = existing_domain.get()
else:
# Raise an err for now
raise Exception(f"Domain {existing_domain} wants to be added but doesn't exist in the DB")
raise Exception(
f"Domain {existing_domain} wants to be added but doesn't exist in the DB"
)
invitation.save()
valid_org_choices = [
(name, value)
for name, value in DomainApplication.OrganizationChoices.choices
]
valid_fed_choices = [
value for name, value in DomainApplication.BranchChoices.choices
]
valid_agency_choices = DomainApplication.AGENCIES
# ======================================================
# ================= DOMAIN INFORMATION =================
logger.info(
@ -661,31 +696,54 @@ class Command(BaseCommand):
{TerminalColors.ENDC}"""
)
for transition_domain in TransitionDomain.objects.all():
target_domain_information, associated_domain, was_created = self.update_or_create_domain_information(transition_domain, debug_on)
(
target_domain_information,
associated_domain,
was_created,
) = self.update_or_create_domain_information(
transition_domain,
valid_agency_choices,
valid_fed_choices,
valid_org_choices,
debug_on,
)
debug_string = ""
if target_domain_information is None:
# ---------------- SKIPPED ----------------
skipped_domain_information_entries.append(target_domain_information)
debug_string = f"skipped domain information: {target_domain_information}"
debug_string = (
f"skipped domain information: {target_domain_information}"
)
elif was_created:
# DEBUG:
TerminalHelper.print_conditional(
debug_on,
(f"{TerminalColors.OKCYAN}"
(
f"{TerminalColors.OKCYAN}"
f"Checking duplicates for: {target_domain_information}"
f"{TerminalColors.ENDC}"), # noqa
f"{TerminalColors.ENDC}"
), # noqa
)
# ---------------- DUPLICATE ----------------
# The unique key constraint does not allow multiple domain
# information objects to share the same domain
existing_domain_information_in_to_create = next(
(x for x in domain_information_to_create if x.domain.name == target_domain_information.domain.name),
(
x
for x in domain_information_to_create
if x.domain.name == target_domain_information.domain.name
),
None,
)
# TODO: this is redundant. Currently debugging....running into unique key constraint error....
existing_domain_info = DomainInformation.objects.filter(domain__name=target_domain_information.domain.name).exists()
if existing_domain_information_in_to_create is not None or existing_domain_info:
existing_domain_info = DomainInformation.objects.filter(
domain__name=target_domain_information.domain.name
).exists()
if (
existing_domain_information_in_to_create is not None
or existing_domain_info
):
debug_string = f"""{TerminalColors.YELLOW}
Duplicate Detected: {existing_domain_information_in_to_create}.
Cannot add duplicate Domain Information object
@ -693,11 +751,15 @@ class Command(BaseCommand):
else:
# ---------------- CREATED ----------------
domain_information_to_create.append(target_domain_information)
debug_string = f"created domain information: {target_domain_information}"
debug_string = (
f"created domain information: {target_domain_information}"
)
elif not was_created:
# ---------------- UPDATED ----------------
updated_domain_information.append(target_domain_information)
debug_string = f"updated domain information: {target_domain_information}"
debug_string = (
f"updated domain information: {target_domain_information}"
)
else:
debug_string = f"domain information already exists and matches incoming data (NO CHANGES MADE): {target_domain_information}"
@ -714,9 +776,11 @@ class Command(BaseCommand):
TerminalHelper.print_conditional(
debug_on,
(f"{TerminalColors.YELLOW}"
(
f"{TerminalColors.YELLOW}"
f"Trying to add: {domain_information_to_create}"
f"{TerminalColors.ENDC}"),
f"{TerminalColors.ENDC}"
),
)
DomainInformation.objects.bulk_create(domain_information_to_create)

25
src/registrar/management/commands/utility/epp_data_containers.py
View file

@ -89,27 +89,12 @@ class EnumFilenames(Enum):
# We are sourcing data from many different locations, so its better to track this
# as an Enum rather than multiple spread out variables.
# We store the "type" as [0], and we store the "default_filepath" as [1].
AGENCY_ADHOC = (
"agency_adhoc",
"agency.adhoc.dotgov.txt"
)
AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt")
DOMAIN_ADDITIONAL = (
"domain_additional",
"domainadditionaldatalink.adhoc.dotgov.txt",
)
DOMAIN_ESCROW = (
"domain_escrow",
"escrow_domains.daily.dotgov.GOV.txt"
)
DOMAIN_ADHOC = (
"domain_adhoc",
"domaintypes.adhoc.dotgov.txt"
)
ORGANIZATION_ADHOC = (
"organization_adhoc",
"organization.adhoc.dotgov.txt"
)
AUTHORITY_ADHOC = (
"authority_adhoc",
"authority.adhoc.dotgov.txt"
)
DOMAIN_ESCROW = ("domain_escrow", "escrow_domains.daily.dotgov.GOV.txt")
DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt")
ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt")
AUTHORITY_ADHOC = ("authority_adhoc", "authority.adhoc.dotgov.txt")

90
src/registrar/management/commands/utility/extra_transition_domain_helper.py
View file

@ -70,9 +70,14 @@ class FileTransitionLog:
dict_name = (file_type, domain_name)
self._add_to_log_list(dict_name, log)
def create_log_item(
self, file_type, code, message, domain_name=None, add_to_list=True, minimal_logging=True
self,
file_type,
code,
message,
domain_name=None,
add_to_list=True,
minimal_logging=True,
):
"""Creates and returns an LogItem object.
@ -107,9 +112,7 @@ class FileTransitionLog:
for parent_log in self.logs:
for child_log in parent_log:
TerminalHelper.print_conditional(
True,
child_log.message,
child_log.severity
True, child_log.message, child_log.severity
)
def display_logs_by_domain_name(self, domain_name, restrict_type=LogCode.DEFAULT):
@ -128,9 +131,7 @@ class FileTransitionLog:
for log in domain_logs:
TerminalHelper.print_conditional(
restrict_type != log.code,
log.message,
log.code
restrict_type != log.code, log.message, log.code
)
def get_logs(self, file_type, domain_name):
@ -213,19 +214,20 @@ class LoadExtraTransitionDomain:
logger.info(
f"""{TerminalColors.OKGREEN}
============= FINISHED ===============
Updated {len(updated_transition_domains)} transition domain entries:
{[domain for domain in updated_transition_domains]}
Updated {len(updated_transition_domains)} transition domain entries
{TerminalColors.ENDC}
"""
)
else:
# TODO - update
TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}")
TerminalHelper.print_conditional(
self.debug,
f"{TerminalHelper.array_as_string(updated_transition_domains)}",
)
logger.error(
f"""{TerminalColors.FAIL}
============= FINISHED WITH ERRORS ===============
Updated {len(updated_transition_domains)} transition domain entries:
{[domain for domain in updated_transition_domains]}
Updated {len(updated_transition_domains)} transition domain entries,
Failed to update {failed_count} transition domain entries:
{[domain for domain in failed_transition_domains]}
{TerminalColors.ENDC}
@ -237,7 +239,8 @@ class LoadExtraTransitionDomain:
total_transition_domains = len(updated_transition_domains)
total_updates_made = TransitionDomain.objects.all().count()
if total_transition_domains != total_updates_made:
logger.error(f"""{TerminalColors.FAIL}
logger.error(
f"""{TerminalColors.FAIL}
WARNING: something went wrong processing domain information data.
Total Transition Domains expecting a data update: {total_transition_domains}
@ -248,7 +251,8 @@ class LoadExtraTransitionDomain:
corrupt data. Please check logs to diagnose.
----- TERMINATING ----
""")
"""
)
sys.exit()
def parse_creation_expiration_data(self, domain_name, transition_domain):
@ -265,16 +269,12 @@ class LoadExtraTransitionDomain:
"Could not add epp_creation_date and epp_expiration_date "
f"on {domain_name}, no data exists.",
domain_name,
not self.debug
not self.debug,
)
return transition_domain
creation_exists = (
transition_domain.epp_creation_date is not None
)
expiration_exists = (
transition_domain.epp_expiration_date is not None
)
creation_exists = transition_domain.epp_creation_date is not None
expiration_exists = transition_domain.epp_expiration_date is not None
transition_domain.epp_creation_date = info.creationdate
transition_domain.epp_expiration_date = info.expirationdate
@ -311,7 +311,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR,
f"Could not add federal_agency on {domain_name}, no data exists.",
domain_name,
not self.debug
not self.debug,
)
return transition_domain
@ -326,7 +326,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR,
f"Could not add inactive agency {info.agencyname} on {domain_name}",
domain_name,
not self.debug
not self.debug,
)
return transition_domain
@ -336,7 +336,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR,
f"Could not add non-federal agency {info.agencyname} on {domain_name}",
domain_name,
not self.debug
not self.debug,
)
return transition_domain
@ -369,7 +369,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR,
f"Could not add domain_type on {domain_name}, no data exists.",
domain_name,
not self.debug
not self.debug,
)
return transition_domain
@ -392,7 +392,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR,
f"Could not add inactive domain_type {domain_type[0]} on {domain_name}",
domain_name,
not self.debug
not self.debug,
)
return transition_domain
@ -453,7 +453,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR,
f"Could not add organization_name on {domain_name}, no data exists.",
domain_name,
not self.debug
not self.debug,
)
return transition_domain
@ -487,7 +487,7 @@ class LoadExtraTransitionDomain:
LogCode.INFO,
f"Added {var_name} as '{changed_value}' on {domain_name}",
domain_name,
not self.debug
not self.debug,
)
else:
self.parse_logs.create_log_item(
@ -495,7 +495,7 @@ class LoadExtraTransitionDomain:
LogCode.WARNING,
f"Updated existing {var_name} to '{changed_value}' on {domain_name}",
domain_name,
not self.debug
not self.debug,
)
# Property getters, i.e. orgid or domaintypeid
@ -615,7 +615,9 @@ class LoadExtraTransitionDomain:
desired_type = self.parsed_data_container.file_data.get(file_type)
if desired_type is None:
self.parse_logs.create_log_item(
file_type, LogCode.ERROR, f"Type {file_type} does not exist",
file_type,
LogCode.ERROR,
f"Type {file_type} does not exist",
)
return None
@ -624,10 +626,13 @@ class LoadExtraTransitionDomain:
obj = desired_type.data.get(desired_id)
if obj is None:
self.parse_logs.create_log_item(
file_type, LogCode.ERROR, f"Id {desired_id} does not exist for {file_type.value[0]}"
file_type,
LogCode.ERROR,
f"Id {desired_id} does not exist for {file_type.value[0]}",
)
return obj
# TODO - change name
@dataclass
class FileDataHolder:
@ -898,26 +903,15 @@ class ExtraTransitionDomain:
file_type.data = {}
def parse_csv_file(
self,
file,
seperator,
dataclass_type,
id_field,
is_domain_escrow=False
self, file, seperator, dataclass_type, id_field, is_domain_escrow=False
):
# Domain escrow is an edge case
if is_domain_escrow:
item_to_return = self._read_domain_escrow(
file,
seperator
)
item_to_return = self._read_domain_escrow(file, seperator)
return item_to_return
else:
item_to_return = self._read_csv_file(
file,
seperator,
dataclass_type,
id_field
file, seperator, dataclass_type, id_field
)
return item_to_return
@ -953,7 +947,9 @@ class ExtraTransitionDomain:
f"Found bad data in {file}. Attempting to clean."
f"{TerminalColors.ENDC}"
)
updated_file_content = self.replace_bad_seperators(file, f"{seperator}", ";badseperator;")
updated_file_content = self.replace_bad_seperators(
file, f"{seperator}", ";badseperator;"
)
dict_data = {}
break

52
src/registrar/management/commands/utility/terminal_helper.py
View file

@ -4,6 +4,7 @@ import sys
logger = logging.getLogger(__name__)
class LogCode(Enum):
"""Stores the desired log severity
@ -21,6 +22,7 @@ class LogCode(Enum):
DEBUG = 4
DEFAULT = 5
class TerminalColors:
"""Colors for terminal outputs
(makes reading the logs WAY easier)"""
@ -81,7 +83,14 @@ class TerminalHelper:
The "answer" return value is True for "yes" or False for "no".
"""
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit"}
valid = {
"yes": True,
"y": True,
"ye": True,
"no": False,
"n": False,
"e": "exit",
}
if default is None:
prompt = " [y/n] "
elif default == "yes":
@ -105,16 +114,14 @@ class TerminalHelper:
# @staticmethod
def array_as_string(array_to_convert: []) -> str:
array_as_string = "{}".format(
"\n".join(map(str, array_to_convert))
)
array_as_string = "{}".format("\n".join(map(str, array_to_convert)))
return array_as_string
@staticmethod
def print_conditional(
print_condition: bool,
print_statement: str,
log_severity: LogCode = LogCode.DEFAULT
log_severity: LogCode = LogCode.DEFAULT,
):
"""This function reduces complexity of debug statements
in other functions.
@ -181,29 +188,40 @@ class TerminalHelper:
@staticmethod
def get_file_line_count(filepath: str) -> int:
with open(filepath,'r') as file:
with open(filepath, "r") as file:
li = file.readlines()
total_line = len(li)
return total_line
@staticmethod
def print_to_file_conditional(print_condition: bool, filename: str, file_directory: str, file_contents: str):
"""Sometimes logger outputs get insanely huge.
"""
if (print_condition):
def print_to_file_conditional(
print_condition: bool, filename: str, file_directory: str, file_contents: str
):
"""Sometimes logger outputs get insanely huge."""
if print_condition:
# Add a slash if the last character isn't one
if file_directory and file_directory[-1] != "/":
file_directory += "/"
# Assemble filepath
filepath = f"{file_directory}{filename}.txt"
# Write to file
logger.info(f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}")
logger.info(
f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}"
)
with open(f"{filepath}", "w+") as f:
f.write(file_contents)
@staticmethod
def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 100, fill = '█', printEnd = "\r"):
def printProgressBar(
iteration,
total,
prefix="Progress:",
suffix="Complete",
decimals=1,
length=100,
fill="█",
printEnd="\r",
):
"""
Call in a loop to create terminal progress bar
@params:
@ -227,10 +245,12 @@ class TerminalHelper:
printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50)
"""
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
percent = ("{0:." + str(decimals) + "f}").format(
100 * (iteration / float(total))
)
filledLength = int(length * iteration // total)
bar = fill * filledLength + '-' * (length - filledLength)
print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd)
bar = fill * filledLength + "-" * (length - filledLength)
print(f"\r{prefix} |{bar}| {percent}% {suffix}", end=printEnd)
# Print New Line on Complete
if iteration == total:
print()

25
src/registrar/management/commands/utility/transition_domain_arguments.py
View file

@ -3,6 +3,7 @@ from typing import Optional
from registrar.management.commands.utility.epp_data_containers import EnumFilenames
@dataclass
class TransitionDomainArguments:
"""Stores arguments for load_transition_domain, structurally a mix
@ -36,14 +37,26 @@ class TransitionDomainArguments:
# Filenames #
## Adhocs ##
agency_adhoc_filename: Optional[str] = field(default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True)
domain_adhoc_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True)
organization_adhoc_filename: Optional[str] = field(default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True)
authority_adhoc_filename: Optional[str] = field(default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True)
agency_adhoc_filename: Optional[str] = field(
default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True
)
domain_adhoc_filename: Optional[str] = field(
default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True
)
organization_adhoc_filename: Optional[str] = field(
default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True
)
authority_adhoc_filename: Optional[str] = field(
default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True
)
## Data files ##
domain_escrow_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True)
domain_additional_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True)
domain_escrow_filename: Optional[str] = field(
default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True
)
domain_additional_filename: Optional[str] = field(
default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True
)
domain_contacts_filename: Optional[str] = field(default=None, repr=True)
domain_statuses_filename: Optional[str] = field(default=None, repr=True)
contacts_filename: Optional[str] = field(default=None, repr=True)

27
src/registrar/tests/test_transition_domain_migrations.py
View file

@ -13,6 +13,7 @@ from registrar.models import (
from django.core.management import call_command
from unittest.mock import patch
class TestMigrations(TestCase):
def setUp(self):
""" """
@ -46,24 +47,30 @@ class TestMigrations(TestCase):
UserDomainRole.objects.all().delete()
def run_load_domains(self):
with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True):
with patch(
"registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit",
return_value=True,
):
call_command(
"load_transition_domain",
self.migration_json_filename,
directory=self.test_data_file_location
directory=self.test_data_file_location,
)
def run_transfer_domains(self):
call_command("transfer_transition_domains_to_domains")
def run_master_script(self):
with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True):
with patch(
"registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit",
return_value=True,
):
call_command(
"master_domain_migrations",
runMigrations=True,
migrationDirectory=self.test_data_file_location,
migrationJSON=self.migration_json_filename,
disablePrompts=True
disablePrompts=True,
)
def compare_tables(
@ -242,7 +249,7 @@ class TestMigrations(TestCase):
federal_type="Executive",
federal_agency="InnoZ",
epp_creation_date=None,
epp_expiration_date=None
epp_expiration_date=None,
),
TransitionDomain(
username="reginald.ratcliff4@test.com",
@ -254,11 +261,13 @@ class TestMigrations(TestCase):
federal_type=None,
federal_agency=None,
epp_creation_date=None,
epp_expiration_date=None
)
epp_expiration_date=None,
),
]
expected_transition_domains = TransitionDomain.objects.filter(username="alexandra.bobbitt5@test.com")
expected_transition_domains = TransitionDomain.objects.filter(
username="alexandra.bobbitt5@test.com"
)
self.assertEqual(expected_transition_domains.count(), 1)
expected_transition_domain = expected_transition_domains.get()
@ -267,7 +276,6 @@ class TestMigrations(TestCase):
all_transition_domains = TransitionDomain.objects.all()
for domain in all_transition_domains:
for expected in expected_transition_domains:
# This data gets created when the object is,
# so we should just match it. Not relevant
# to the added data.
@ -386,7 +394,6 @@ class TestMigrations(TestCase):
self.assertEqual(Users.count(), 1)
self.assertEqual(anomaly.creator, Users.get())
def test_transfer_transition_domains_to_domains(self):
self.run_load_domains()
self.run_transfer_domains()