zydronium/manage.get.gov
1
0
Fork 0
mirror of https://github.com/cisagov/manage.get.gov.git synced 2025-05-22 12:29:25 +02:00
Code Issues Projects Releases Packages Wiki Activity

Linting + minimize logging to a degree

Browse source
This commit is contained in:
zandercymatics 2023-11-08 17:37:59 -07:00
parent 7d798e0e43
commit 72d95f6fad
No known key found for this signature in database
GPG key ID: FF4636ABEC9682B7
11 changed files with 507 additions and 373 deletions
Download patch file Download diff file Expand all files Collapse all files

18
docs/operations/data_migration.md
View file

@ -240,7 +240,23 @@ This will allow Docker to mount the files to a container (under `/app`) for our
### STEP 1: Load Transition Domains ### STEP 1: Load Transition Domains
Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table. Run the following command, making sure the file paths point to the right location. This will parse the three given files and load the information into the TransitionDomain table.
##### Create a JSON file
In your chosen directory (either `src/tmp` or `src/migrationdata` depending on preference), create a json file called `migrationFilepaths.json`. This file will map to other urls
Example
```
{
"directory": "migrationdata/",
"agency_adhoc_filename": "20231009.agency.adhoc.dotgov.txt",
"authority_adhoc_filename": "authority.adhoc.dotgov.txt",
"contacts_filename": "escrow_contacts.daily.dotgov.GOV.txt",
"domain_adhoc_filename": "20231009.domaintypes.adhoc.dotgov.txt",
"domain_additional_filename": "20231009.domainadditionaldatalink.adhoc.dotgov.txt",
"domain_contacts_filename": "escrow_domain_contacts.daily.dotgov.GOV.txt",
"domain_escrow_filename": "escrow_domains.daily.dotgov.GOV.txt",
"domain_statuses_filename": "escrow_domain_statuses.daily.dotgov.GOV.txt",
"organization_adhoc_filename": "20231009.organization.adhoc.dotgov.txt"
}
```
##### LOCAL COMMAND ##### LOCAL COMMAND
```shell ```shell
docker-compose exec app ./manage.py load_transition_domain migrationFilepaths.json --directory /app/tmp/ --debug --limitParse 10 docker-compose exec app ./manage.py load_transition_domain migrationFilepaths.json --directory /app/tmp/ --debug --limitParse 10

111
src/registrar/management/commands/agency_data_extractor.py
View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
# Example command for running this script: # Example command for running this script:
# docker compose run -T app ./manage.py agency_data_extractor 20231009.agency.adhoc.dotgov.txt --dir /app/tmp --debug # docker compose run -T app ./manage.py agency_data_extractor 20231009.agency.adhoc.dotgov.txt --dir /app/tmp --debug
class Command(BaseCommand): class Command(BaseCommand):
help = """Loads data for domains that are in transition help = """Loads data for domains that are in transition
(populates transition_domain model objects).""" (populates transition_domain model objects)."""
@ -26,38 +27,40 @@ class Command(BaseCommand):
parser.add_argument( parser.add_argument(
"agency_data_filename", help="Data file with agency information" "agency_data_filename", help="Data file with agency information"
) )
parser.add_argument( parser.add_argument("--dir", default="migrationdata", help="Desired directory")
"--dir", default="migrationdata", help="Desired directory"
)
parser.add_argument("--sep", default="|", help="Delimiter character") parser.add_argument("--sep", default="|", help="Delimiter character")
parser.add_argument("--debug", help="Prints additional debug statements to the terminal", action=argparse.BooleanOptionalAction) parser.add_argument(
"--debug",
help="Prints additional debug statements to the terminal",
action=argparse.BooleanOptionalAction,
)
parser.add_argument("--prompt", action=argparse.BooleanOptionalAction) parser.add_argument("--prompt", action=argparse.BooleanOptionalAction)
@staticmethod @staticmethod
def extract_agencies( def extract_agencies(agency_data_filepath: str, sep: str, debug: bool) -> [str]:
agency_data_filepath: str,
sep: str,
debug: bool
) -> [str]:
"""Extracts all the agency names from the provided """Extracts all the agency names from the provided
agency file (skips any duplicates) and returns those agency file (skips any duplicates) and returns those
names in an array""" names in an array"""
agency_names = [] agency_names = []
logger.info(f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}") logger.info(
f"{TerminalColors.OKCYAN}Reading agency data file {agency_data_filepath}{TerminalColors.ENDC}"
)
with open(agency_data_filepath, "r") as agency_data_filepath: # noqa with open(agency_data_filepath, "r") as agency_data_filepath: # noqa
for row in csv.reader(agency_data_filepath, delimiter=sep): for row in csv.reader(agency_data_filepath, delimiter=sep):
agency_name = row[1] agency_name = row[1]
TerminalHelper.print_conditional(debug, f"Checking: {agency_name}") TerminalHelper.print_conditional(debug, f"Checking: {agency_name}")
if agency_name not in agency_names: if agency_name not in agency_names:
agency_names.append(agency_name) agency_names.append(agency_name)
logger.info(f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}") logger.info(
f"{TerminalColors.OKCYAN}Checked {len(agency_names)} agencies{TerminalColors.ENDC}"
)
return agency_names return agency_names
@staticmethod @staticmethod
def compare_agency_lists(provided_agencies: [str], def compare_agency_lists(
existing_agencies: [str], provided_agencies: [str], existing_agencies: [str], debug: bool
debug: bool): ):
""" """
Compares new_agencies with existing_agencies and Compares new_agencies with existing_agencies and
provides the equivalent of an outer-join on the two provides the equivalent of an outer-join on the two
@ -69,27 +72,37 @@ class Command(BaseCommand):
for agency in provided_agencies: for agency in provided_agencies:
if agency not in existing_agencies and agency not in new_agencies: if agency not in existing_agencies and agency not in new_agencies:
new_agencies.append(agency) new_agencies.append(agency)
TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}") TerminalHelper.print_conditional(
debug,
f"{TerminalColors.YELLOW}Found new agency: {agency}{TerminalColors.ENDC}",
)
possibly_unused_agencies = [] possibly_unused_agencies = []
# 2 - Get all new agencies that we don't already have (We might want to ADD these to our list) # 2 - Get all new agencies that we don't already have (We might want to ADD these to our list)
for agency in existing_agencies: for agency in existing_agencies:
if agency not in provided_agencies and agency not in possibly_unused_agencies: if (
agency not in provided_agencies
and agency not in possibly_unused_agencies
):
possibly_unused_agencies.append(agency) possibly_unused_agencies.append(agency)
TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}") TerminalHelper.print_conditional(
debug,
f"{TerminalColors.YELLOW}Possibly unused agency detected: {agency}{TerminalColors.ENDC}",
)
matched_agencies = [] matched_agencies = []
for agency in provided_agencies: for agency in provided_agencies:
if agency in existing_agencies: if agency in existing_agencies:
matched_agencies.append(agency) matched_agencies.append(agency)
TerminalHelper.print_conditional(debug, f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}") TerminalHelper.print_conditional(
debug,
f"{TerminalColors.YELLOW}Matched agencies: {agency}{TerminalColors.ENDC}",
)
# Print the summary of findings # Print the summary of findings
# 1 - Print the list of agencies in the NEW list, which we do not already have # 1 - Print the list of agencies in the NEW list, which we do not already have
# 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies? # 2 - Print the list of agencies that we currently have, which are NOT in the new list (these might be eligible for removal?) TODO: would we ever want to remove existing agencies?
new_agencies_as_string = "{}".format( new_agencies_as_string = "{}".format(",\n ".join(map(str, new_agencies)))
",\n ".join(map(str, new_agencies))
)
possibly_unused_agencies_as_string = "{}".format( possibly_unused_agencies_as_string = "{}".format(
",\n ".join(map(str, possibly_unused_agencies)) ",\n ".join(map(str, possibly_unused_agencies))
) )
@ -97,7 +110,8 @@ class Command(BaseCommand):
",\n ".join(map(str, matched_agencies)) ",\n ".join(map(str, matched_agencies))
) )
logger.info(f""" logger.info(
f"""
{TerminalColors.OKGREEN} {TerminalColors.OKGREEN}
======================== SUMMARY OF FINDINGS ============================ ======================== SUMMARY OF FINDINGS ============================
{len(provided_agencies)} AGENCIES WERE PROVIDED in the agency file. {len(provided_agencies)} AGENCIES WERE PROVIDED in the agency file.
@ -117,13 +131,12 @@ class Command(BaseCommand):
These agencies are in our system, but not in the provided agency file: These agencies are in our system, but not in the provided agency file:
{TerminalColors.YELLOW}{possibly_unused_agencies_as_string} {TerminalColors.YELLOW}{possibly_unused_agencies_as_string}
{TerminalColors.ENDC} {TerminalColors.ENDC}
""") """
)
@staticmethod @staticmethod
def print_agency_list(agencies, filename): def print_agency_list(agencies, filename):
full_agency_list_as_string = "{}".format( full_agency_list_as_string = "{}".format(",\n".join(map(str, agencies)))
",\n".join(map(str, agencies))
)
logger.info( logger.info(
f"\n{TerminalColors.YELLOW}" f"\n{TerminalColors.YELLOW}"
f"\n{full_agency_list_as_string}" f"\n{full_agency_list_as_string}"
@ -146,14 +159,17 @@ class Command(BaseCommand):
prompt = options.get("prompt") prompt = options.get("prompt")
dir = options.get("dir") dir = options.get("dir")
agency_data_file = dir+"/"+agency_data_filename agency_data_file = dir + "/" + agency_data_filename
new_agencies = self.extract_agencies(agency_data_file, sep, debug) new_agencies = self.extract_agencies(agency_data_file, sep, debug)
hard_coded_agencies = DomainApplication.AGENCIES hard_coded_agencies = DomainApplication.AGENCIES
transition_domain_agencies = TransitionDomain.objects.all().values_list('federal_agency', flat=True).distinct() transition_domain_agencies = (
TransitionDomain.objects.all()
.values_list("federal_agency", flat=True)
.distinct()
)
print(transition_domain_agencies) print(transition_domain_agencies)
merged_agencies = new_agencies merged_agencies = new_agencies
for agency in hard_coded_agencies: for agency in hard_coded_agencies:
if agency not in merged_agencies: if agency not in merged_agencies:
@ -168,19 +184,25 @@ class Command(BaseCommand):
# OPTION to compare the agency file to our hard-coded list # OPTION to compare the agency file to our hard-coded list
if prompt: if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}") prompt_successful = TerminalHelper.query_yes_no(
f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against our (hard-coded) dropdown list of agencies?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt: if prompt_successful or not prompt:
self.compare_agency_lists(new_agencies, hard_coded_agencies, debug) self.compare_agency_lists(new_agencies, hard_coded_agencies, debug)
# OPTION to compare the agency file to Transition Domains # OPTION to compare the agency file to Transition Domains
if prompt: if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}") prompt_successful = TerminalHelper.query_yes_no(
f"\n\n{TerminalColors.FAIL}Check {agency_data_filename} against Transition Domain contents?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt: if prompt_successful or not prompt:
self.compare_agency_lists(new_agencies, transition_domain_agencies, debug) self.compare_agency_lists(new_agencies, transition_domain_agencies, debug)
# OPTION to print out the full list of agencies from the agency file # OPTION to print out the full list of agencies from the agency file
if prompt: if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}") prompt_successful = TerminalHelper.query_yes_no(
f"\n\n{TerminalColors.FAIL}Would you like to print the full list of agencies from the given agency file?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt: if prompt_successful or not prompt:
logger.info( logger.info(
f"\n{TerminalColors.OKGREEN}" f"\n{TerminalColors.OKGREEN}"
@ -192,7 +214,9 @@ class Command(BaseCommand):
# OPTION to print out the full list of agencies from the agency file # OPTION to print out the full list of agencies from the agency file
if prompt: if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}") prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt: if prompt_successful or not prompt:
logger.info( logger.info(
f"\n{TerminalColors.OKGREEN}" f"\n{TerminalColors.OKGREEN}"
@ -204,7 +228,9 @@ class Command(BaseCommand):
# OPTION to print out the full list of agencies from the agency file # OPTION to print out the full list of agencies from the agency file
if prompt: if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}") prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the full list of agencies from the dropdown?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt: if prompt_successful or not prompt:
logger.info( logger.info(
f"\n{TerminalColors.OKGREEN}" f"\n{TerminalColors.OKGREEN}"
@ -212,12 +238,15 @@ class Command(BaseCommand):
f"\nThese are all the agencies in the Transition Domains table." f"\nThese are all the agencies in the Transition Domains table."
f"\n\n{len(transition_domain_agencies)} TOTAL\n\n" f"\n\n{len(transition_domain_agencies)} TOTAL\n\n"
) )
self.print_agency_list(transition_domain_agencies, "Transition_Domain_Agencies") self.print_agency_list(
transition_domain_agencies, "Transition_Domain_Agencies"
)
# OPTION to print out the full list of agencies from the agency file # OPTION to print out the full list of agencies from the agency file
if prompt: if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}") prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt: if prompt_successful or not prompt:
logger.info( logger.info(
f"\n{TerminalColors.OKGREEN}" f"\n{TerminalColors.OKGREEN}"
@ -229,7 +258,9 @@ class Command(BaseCommand):
# OPTION to print out the full list of agencies from the agency file # OPTION to print out the full list of agencies from the agency file
if prompt: if prompt:
prompt_successful = TerminalHelper.query_yes_no(f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}") prompt_successful = TerminalHelper.query_yes_no(
f"{TerminalColors.FAIL}Would you like to print the MERGED list of agencies (dropdown + agency file)?{TerminalColors.ENDC}"
)
if prompt_successful or not prompt: if prompt_successful or not prompt:
logger.info( logger.info(
f"\n{TerminalColors.OKGREEN}" f"\n{TerminalColors.OKGREEN}"
@ -237,4 +268,6 @@ class Command(BaseCommand):
f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file." f"\nThese are all the agencies our transition domains table plus all the agencies in the agency file."
f"\n\n{len(merged_agencies)} TOTAL\n\n" f"\n\n{len(merged_agencies)} TOTAL\n\n"
) )
self.print_agency_list(merged_transition_agencies, "Merged_Transition_Domain_Agency_List") self.print_agency_list(
merged_transition_agencies, "Merged_Transition_Domain_Agency_List"
)

27
src/registrar/management/commands/load_transition_domain.py
View file

@ -66,7 +66,7 @@ class Command(BaseCommand):
parser.add_argument( parser.add_argument(
"--infer_filenames", "--infer_filenames",
action=argparse.BooleanOptionalAction, action=argparse.BooleanOptionalAction,
help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting." help="Determines if we should infer filenames or not. Recommended to be enabled only in a development or testing setting.",
) )
parser.add_argument( parser.add_argument(
@ -74,7 +74,7 @@ class Command(BaseCommand):
) )
parser.add_argument( parser.add_argument(
"--domain_contacts_filename", "--domain_contacts_filename",
help="Data file with domain contact information" help="Data file with domain contact information",
) )
parser.add_argument( parser.add_argument(
"--contacts_filename", "--contacts_filename",
@ -82,7 +82,7 @@ class Command(BaseCommand):
) )
parser.add_argument( parser.add_argument(
"--domain_statuses_filename", "--domain_statuses_filename",
help="Data file with domain status information" help="Data file with domain status information",
) )
parser.add_argument( parser.add_argument(
"--agency_adhoc_filename", "--agency_adhoc_filename",
@ -347,10 +347,12 @@ class Command(BaseCommand):
# If it does, update the options # If it does, update the options
options[key] = value options[key] = value
except Exception as err: except Exception as err:
logger.error(f"""{TerminalColors.FAIL}There was an error loading the JSON responsible logger.error(
f"""{TerminalColors.FAIL}There was an error loading the JSON responsible
for providing filepaths. for providing filepaths.
{TerminalColors.ENDC} {TerminalColors.ENDC}
""") """
)
raise err raise err
sep = args.sep sep = args.sep
@ -370,11 +372,12 @@ class Command(BaseCommand):
## Variables for Additional TransitionDomain Information ## ## Variables for Additional TransitionDomain Information ##
# Main script filenames - these do not have defaults # Main script filenames - these do not have defaults
domain_contacts_filename = None domain_contacts_filename = None
try: try:
domain_contacts_filename = directory + options.get("domain_contacts_filename") domain_contacts_filename = directory + options.get(
"domain_contacts_filename"
)
except TypeError as err: except TypeError as err:
logger.error( logger.error(
f"Invalid filename of '{args.domain_contacts_filename}'" f"Invalid filename of '{args.domain_contacts_filename}'"
@ -392,7 +395,9 @@ class Command(BaseCommand):
domain_statuses_filename = None domain_statuses_filename = None
try: try:
domain_statuses_filename = directory + options.get("domain_statuses_filename") domain_statuses_filename = directory + options.get(
"domain_statuses_filename"
)
except TypeError as err: except TypeError as err:
logger.error( logger.error(
f"Invalid filename of '{args.domain_statuses_filename}'" f"Invalid filename of '{args.domain_statuses_filename}'"
@ -468,7 +473,10 @@ class Command(BaseCommand):
new_entry_email = "" new_entry_email = ""
new_entry_emailSent = False # set to False by default new_entry_emailSent = False # set to False by default
TerminalHelper.print_conditional(True, f"Processing item {total_rows_parsed}: {new_entry_domain_name}") TerminalHelper.print_conditional(
True,
f"Processing item {total_rows_parsed}: {new_entry_domain_name}",
)
# PART 1: Get the status # PART 1: Get the status
if new_entry_domain_name not in domain_status_dictionary: if new_entry_domain_name not in domain_status_dictionary:
@ -608,7 +616,6 @@ class Command(BaseCommand):
) )
self.print_summary_status_findings(domains_without_status, outlier_statuses) self.print_summary_status_findings(domains_without_status, outlier_statuses)
logger.info( logger.info(
f"""{TerminalColors.OKGREEN} f"""{TerminalColors.OKGREEN}
============= FINISHED =============== ============= FINISHED ===============

11
src/registrar/management/commands/master_domain_migrations.py
View file

@ -30,7 +30,6 @@ logger = logging.getLogger(__name__)
class Command(BaseCommand): class Command(BaseCommand):
help = """ """ # TODO: update this! help = """ """ # TODO: update this!
# ====================================================== # ======================================================
# ================== ARGUMENTS =================== # ================== ARGUMENTS ===================
# ====================================================== # ======================================================
@ -129,7 +128,6 @@ class Command(BaseCommand):
action=argparse.BooleanOptionalAction, action=argparse.BooleanOptionalAction,
) )
# ====================================================== # ======================================================
# =============== DATA ANALYSIS ================== # =============== DATA ANALYSIS ==================
# ====================================================== # ======================================================
@ -252,7 +250,6 @@ class Command(BaseCommand):
""" """
) )
# ====================================================== # ======================================================
# ================= MIGRATIONS =================== # ================= MIGRATIONS ===================
# ====================================================== # ======================================================
@ -266,17 +263,13 @@ class Command(BaseCommand):
prompts_enabled: bool, prompts_enabled: bool,
debug_max_entries_to_parse: int, debug_max_entries_to_parse: int,
): ):
if file_directory and file_directory[-1] != "/": if file_directory and file_directory[-1] != "/":
file_directory += "/" file_directory += "/"
json_filepath = migration_json_filename json_filepath = migration_json_filename
"""Runs the load_transition_domain script""" """Runs the load_transition_domain script"""
# Create the command string # Create the command string
command_script = "load_transition_domain" command_script = "load_transition_domain"
command_string = ( command_string = f"./manage.py {command_script} " f"{json_filepath} "
f"./manage.py {command_script} "
f"{json_filepath} "
)
if sep is not None and sep != "|": if sep is not None and sep != "|":
command_string += f"--sep {sep} " command_string += f"--sep {sep} "
if reset_table: if reset_table:
@ -306,7 +299,7 @@ class Command(BaseCommand):
resetTable=reset_table, resetTable=reset_table,
debug=debug_on, debug=debug_on,
limitParse=debug_max_entries_to_parse, limitParse=debug_max_entries_to_parse,
directory=file_directory directory=file_directory,
) )
def run_transfer_script(self, debug_on: bool, prompts_enabled: bool): def run_transfer_script(self, debug_on: bool, prompts_enabled: bool):

194
src/registrar/management/commands/transfer_transition_domains_to_domains.py
View file

@ -39,7 +39,6 @@ class Command(BaseCommand):
help="Sets max number of entries to load, set to 0 to load all entries", help="Sets max number of entries to load, set to 0 to load all entries",
) )
# ====================================================== # ======================================================
# ===================== PRINTING ====================== # ===================== PRINTING ======================
# ====================================================== # ======================================================
@ -67,12 +66,13 @@ class Command(BaseCommand):
""", """,
) )
def parse_limit_reached(self, def parse_limit_reached(
debug_max_entries_to_parse: bool, self, debug_max_entries_to_parse: bool, total_rows_parsed: int
total_rows_parsed: int
) -> bool: ) -> bool:
if (debug_max_entries_to_parse > 0 if (
and total_rows_parsed >= debug_max_entries_to_parse): debug_max_entries_to_parse > 0
and total_rows_parsed >= debug_max_entries_to_parse
):
logger.info( logger.info(
f"""{TerminalColors.YELLOW} f"""{TerminalColors.YELLOW}
----PARSE LIMIT REACHED. HALTING PARSER.---- ----PARSE LIMIT REACHED. HALTING PARSER.----
@ -156,14 +156,13 @@ class Command(BaseCommand):
""", """,
) )
# ====================================================== # ======================================================
# =================== DOMAIN ===================== # =================== DOMAIN =====================
# ====================================================== # ======================================================
def update_or_create_domain(self, def update_or_create_domain(
transition_domain: TransitionDomain, self, transition_domain: TransitionDomain, debug_on: bool
debug_on: bool) -> (Domain, bool): ) -> (Domain, bool):
""" Given a transition domain, either finds & updates an existing """Given a transition domain, either finds & updates an existing
corresponding domain, or creates a new corresponding domain in corresponding domain, or creates a new corresponding domain in
the Domain table. the Domain table.
@ -258,7 +257,6 @@ class Command(BaseCommand):
) )
return (target_domain, True) return (target_domain, True)
def update_domain_status( def update_domain_status(
self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool self, transition_domain: TransitionDomain, target_domain: Domain, debug_on: bool
) -> bool: ) -> bool:
@ -289,7 +287,6 @@ class Command(BaseCommand):
return True return True
return False return False
# ====================================================== # ======================================================
# ================ DOMAIN INVITATION ================== # ================ DOMAIN INVITATION ==================
# ====================================================== # ======================================================
@ -336,22 +333,26 @@ class Command(BaseCommand):
# ====================================================== # ======================================================
# ================ DOMAIN INFORMATION ================= # ================ DOMAIN INFORMATION =================
# ====================================================== # ======================================================
def update_domain_information(self, current: DomainInformation, target: DomainInformation, debug_on: bool) -> bool: def update_domain_information(
self, current: DomainInformation, target: DomainInformation, debug_on: bool
) -> bool:
# DEBUG: # DEBUG:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
debug_on, debug_on,
(f"{TerminalColors.OKCYAN}" (
f"{TerminalColors.OKCYAN}"
f"Updating: {current}" f"Updating: {current}"
f"{TerminalColors.ENDC}"), # noqa f"{TerminalColors.ENDC}"
), # noqa
) )
updated = False updated = False
fields_to_update = [ fields_to_update = [
'organization_type', "organization_type",
'federal_type', "federal_type",
'federal_agency', "federal_agency",
"organization_name" "organization_name",
] ]
defaults = {field: getattr(target, field) for field in fields_to_update} defaults = {field: getattr(target, field) for field in fields_to_update}
if current != target: if current != target:
@ -364,10 +365,15 @@ class Command(BaseCommand):
def try_add_domain_information(self): def try_add_domain_information(self):
pass pass
def create_new_domain_info(self, def create_new_domain_info(
self,
transition_domain: TransitionDomain, transition_domain: TransitionDomain,
domain: Domain) -> DomainInformation: domain: Domain,
agency_choices,
fed_choices,
org_choices,
debug_on,
) -> DomainInformation:
org_type = transition_domain.organization_type org_type = transition_domain.organization_type
fed_type = transition_domain.federal_type fed_type = transition_domain.federal_type
fed_agency = transition_domain.federal_agency fed_agency = transition_domain.federal_agency
@ -388,32 +394,31 @@ class Command(BaseCommand):
case "Independent Intrastate": case "Independent Intrastate":
org_type = ("special_district", "Special district") org_type = ("special_district", "Special district")
valid_org_type = org_type in [(name, value) for name, value in DomainApplication.OrganizationChoices.choices] valid_org_type = org_type in org_choices
valid_fed_type = fed_type in [value for name, value in DomainApplication.BranchChoices.choices] valid_fed_type = fed_type in fed_choices
valid_fed_agency = fed_agency in DomainApplication.AGENCIES valid_fed_agency = fed_agency in agency_choices
default_creator, _ = User.objects.get_or_create(username="System") default_creator, _ = User.objects.get_or_create(username="System")
new_domain_info_data = { new_domain_info_data = {
'domain': domain, "domain": domain,
'organization_name': transition_domain.organization_name, "organization_name": transition_domain.organization_name,
"creator": default_creator, "creator": default_creator,
} }
if valid_org_type: if valid_org_type:
new_domain_info_data['organization_type'] = org_type[0] new_domain_info_data["organization_type"] = org_type[0]
else: elif debug_on:
logger.debug(f"No org type found on {domain.name}") logger.debug(f"No org type found on {domain.name}")
if valid_fed_type: if valid_fed_type:
new_domain_info_data['federal_type'] = fed_type.lower() new_domain_info_data["federal_type"] = fed_type.lower()
pass elif debug_on:
else:
logger.debug(f"No federal type found on {domain.name}") logger.debug(f"No federal type found on {domain.name}")
if valid_fed_agency: if valid_fed_agency:
new_domain_info_data['federal_agency'] = fed_agency new_domain_info_data["federal_agency"] = fed_agency
else: elif debug_on:
logger.debug(f"No federal agency found on {domain.name}") logger.debug(f"No federal agency found on {domain.name}")
new_domain_info = DomainInformation(**new_domain_info_data) new_domain_info = DomainInformation(**new_domain_info_data)
@ -421,16 +426,22 @@ class Command(BaseCommand):
# DEBUG: # DEBUG:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
True, True,
(f"{TerminalColors.MAGENTA}" (
f"{TerminalColors.MAGENTA}"
f"Created Domain Information template for: {new_domain_info}" f"Created Domain Information template for: {new_domain_info}"
f"{TerminalColors.ENDC}"), # noqa f"{TerminalColors.ENDC}"
), # noqa
) )
return new_domain_info return new_domain_info
def update_or_create_domain_information(self, def update_or_create_domain_information(
self,
transition_domain: TransitionDomain, transition_domain: TransitionDomain,
debug_on: bool) -> (DomainInformation, bool): agency_choices,
fed_choices,
org_choices,
debug_on: bool,
) -> (DomainInformation, bool):
transition_domain_name = transition_domain.domain_name transition_domain_name = transition_domain.domain_name
# Get associated domain # Get associated domain
@ -444,20 +455,33 @@ class Command(BaseCommand):
) )
return (None, None, False) return (None, None, False)
domain = domain_data.get() domain = domain_data.get()
template_domain_information = self.create_new_domain_info(transition_domain, domain) template_domain_information = self.create_new_domain_info(
transition_domain,
domain,
agency_choices,
fed_choices,
org_choices,
debug_on,
)
target_domain_information = None target_domain_information = None
domain_information_exists = DomainInformation.objects.filter(domain__name=transition_domain_name).exists() domain_information_exists = DomainInformation.objects.filter(
domain__name=transition_domain_name
).exists()
if domain_information_exists: if domain_information_exists:
try: try:
# get the existing domain information object # get the existing domain information object
target_domain_information = DomainInformation.objects.get(domain__name=transition_domain_name) target_domain_information = DomainInformation.objects.get(
domain__name=transition_domain_name
)
# DEBUG: # DEBUG:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
debug_on, debug_on,
(f"{TerminalColors.FAIL}" (
f"{TerminalColors.FAIL}"
f"Found existing entry in Domain Information table for:" f"Found existing entry in Domain Information table for:"
f"{transition_domain_name}" f"{transition_domain_name}"
f"{TerminalColors.ENDC}"), # noqa f"{TerminalColors.ENDC}"
), # noqa
) )
# for existing entry, update the status to # for existing entry, update the status to
@ -493,15 +517,15 @@ class Command(BaseCommand):
# DEBUG: # DEBUG:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
debug_on, debug_on,
(f"{TerminalColors.OKCYAN}" (
f"{TerminalColors.OKCYAN}"
f"Adding domain information for: " f"Adding domain information for: "
f"{transition_domain_name}" f"{transition_domain_name}"
f"{TerminalColors.ENDC}"), f"{TerminalColors.ENDC}"
),
) )
return (target_domain_information, domain, True) return (target_domain_information, domain, True)
# ====================================================== # ======================================================
# ===================== HANDLE ======================== # ===================== HANDLE ========================
# ====================================================== # ======================================================
@ -536,7 +560,6 @@ class Command(BaseCommand):
# domain invitations to ADD # domain invitations to ADD
domain_invitations_to_create = [] domain_invitations_to_create = []
# if we are limiting our parse (for testing purposes, keep # if we are limiting our parse (for testing purposes, keep
# track of total rows parsed) # track of total rows parsed)
total_rows_parsed = 0 total_rows_parsed = 0
@ -574,7 +597,9 @@ class Command(BaseCommand):
# ====================================================== # ======================================================
# ====================== DOMAIN ======================= # ====================== DOMAIN =======================
target_domain, was_created = self.update_or_create_domain(transition_domain, debug_on) target_domain, was_created = self.update_or_create_domain(
transition_domain, debug_on
)
debug_string = "" debug_string = ""
if target_domain is None: if target_domain is None:
@ -633,16 +658,16 @@ class Command(BaseCommand):
if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed): if self.parse_limit_reached(debug_max_entries_to_parse, total_rows_parsed):
break break
# First, save all Domain objects to the database # First, save all Domain objects to the database
Domain.objects.bulk_create(domains_to_create) Domain.objects.bulk_create(domains_to_create)
#DomainInvitation.objects.bulk_create(domain_invitations_to_create) # DomainInvitation.objects.bulk_create(domain_invitations_to_create)
# TODO: this is to resolve an error where bulk_create # TODO: this is to resolve an error where bulk_create
# doesn't save to database in a way that invitation objects can # doesn't save to database in a way that invitation objects can
# utilize. # utilize.
# Then, create DomainInvitation objects # Then, create DomainInvitation objects
for invitation in domain_invitations_to_create: for invitation in domain_invitations_to_create:
if debug_on:
logger.info(f"Pairing invite to its domain...{invitation}") logger.info(f"Pairing invite to its domain...{invitation}")
existing_domain = Domain.objects.filter(name=invitation.domain.name) existing_domain = Domain.objects.filter(name=invitation.domain.name)
# Make sure the related Domain object is saved # Make sure the related Domain object is saved
@ -650,9 +675,19 @@ class Command(BaseCommand):
invitation.domain = existing_domain.get() invitation.domain = existing_domain.get()
else: else:
# Raise an err for now # Raise an err for now
raise Exception(f"Domain {existing_domain} wants to be added but doesn't exist in the DB") raise Exception(
f"Domain {existing_domain} wants to be added but doesn't exist in the DB"
)
invitation.save() invitation.save()
valid_org_choices = [
(name, value)
for name, value in DomainApplication.OrganizationChoices.choices
]
valid_fed_choices = [
value for name, value in DomainApplication.BranchChoices.choices
]
valid_agency_choices = DomainApplication.AGENCIES
# ====================================================== # ======================================================
# ================= DOMAIN INFORMATION ================= # ================= DOMAIN INFORMATION =================
logger.info( logger.info(
@ -661,31 +696,54 @@ class Command(BaseCommand):
{TerminalColors.ENDC}""" {TerminalColors.ENDC}"""
) )
for transition_domain in TransitionDomain.objects.all(): for transition_domain in TransitionDomain.objects.all():
target_domain_information, associated_domain, was_created = self.update_or_create_domain_information(transition_domain, debug_on) (
target_domain_information,
associated_domain,
was_created,
) = self.update_or_create_domain_information(
transition_domain,
valid_agency_choices,
valid_fed_choices,
valid_org_choices,
debug_on,
)
debug_string = "" debug_string = ""
if target_domain_information is None: if target_domain_information is None:
# ---------------- SKIPPED ---------------- # ---------------- SKIPPED ----------------
skipped_domain_information_entries.append(target_domain_information) skipped_domain_information_entries.append(target_domain_information)
debug_string = f"skipped domain information: {target_domain_information}" debug_string = (
f"skipped domain information: {target_domain_information}"
)
elif was_created: elif was_created:
# DEBUG: # DEBUG:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
debug_on, debug_on,
(f"{TerminalColors.OKCYAN}" (
f"{TerminalColors.OKCYAN}"
f"Checking duplicates for: {target_domain_information}" f"Checking duplicates for: {target_domain_information}"
f"{TerminalColors.ENDC}"), # noqa f"{TerminalColors.ENDC}"
), # noqa
) )
# ---------------- DUPLICATE ---------------- # ---------------- DUPLICATE ----------------
# The unique key constraint does not allow multiple domain # The unique key constraint does not allow multiple domain
# information objects to share the same domain # information objects to share the same domain
existing_domain_information_in_to_create = next( existing_domain_information_in_to_create = next(
(x for x in domain_information_to_create if x.domain.name == target_domain_information.domain.name), (
x
for x in domain_information_to_create
if x.domain.name == target_domain_information.domain.name
),
None, None,
) )
# TODO: this is redundant. Currently debugging....running into unique key constraint error.... # TODO: this is redundant. Currently debugging....running into unique key constraint error....
existing_domain_info = DomainInformation.objects.filter(domain__name=target_domain_information.domain.name).exists() existing_domain_info = DomainInformation.objects.filter(
if existing_domain_information_in_to_create is not None or existing_domain_info: domain__name=target_domain_information.domain.name
).exists()
if (
existing_domain_information_in_to_create is not None
or existing_domain_info
):
debug_string = f"""{TerminalColors.YELLOW} debug_string = f"""{TerminalColors.YELLOW}
Duplicate Detected: {existing_domain_information_in_to_create}. Duplicate Detected: {existing_domain_information_in_to_create}.
Cannot add duplicate Domain Information object Cannot add duplicate Domain Information object
@ -693,11 +751,15 @@ class Command(BaseCommand):
else: else:
# ---------------- CREATED ---------------- # ---------------- CREATED ----------------
domain_information_to_create.append(target_domain_information) domain_information_to_create.append(target_domain_information)
debug_string = f"created domain information: {target_domain_information}" debug_string = (
f"created domain information: {target_domain_information}"
)
elif not was_created: elif not was_created:
# ---------------- UPDATED ---------------- # ---------------- UPDATED ----------------
updated_domain_information.append(target_domain_information) updated_domain_information.append(target_domain_information)
debug_string = f"updated domain information: {target_domain_information}" debug_string = (
f"updated domain information: {target_domain_information}"
)
else: else:
debug_string = f"domain information already exists and matches incoming data (NO CHANGES MADE): {target_domain_information}" debug_string = f"domain information already exists and matches incoming data (NO CHANGES MADE): {target_domain_information}"
@ -714,9 +776,11 @@ class Command(BaseCommand):
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
debug_on, debug_on,
(f"{TerminalColors.YELLOW}" (
f"{TerminalColors.YELLOW}"
f"Trying to add: {domain_information_to_create}" f"Trying to add: {domain_information_to_create}"
f"{TerminalColors.ENDC}"), f"{TerminalColors.ENDC}"
),
) )
DomainInformation.objects.bulk_create(domain_information_to_create) DomainInformation.objects.bulk_create(domain_information_to_create)

25
src/registrar/management/commands/utility/epp_data_containers.py
View file

@ -89,27 +89,12 @@ class EnumFilenames(Enum):
# We are sourcing data from many different locations, so its better to track this # We are sourcing data from many different locations, so its better to track this
# as an Enum rather than multiple spread out variables. # as an Enum rather than multiple spread out variables.
# We store the "type" as [0], and we store the "default_filepath" as [1]. # We store the "type" as [0], and we store the "default_filepath" as [1].
AGENCY_ADHOC = ( AGENCY_ADHOC = ("agency_adhoc", "agency.adhoc.dotgov.txt")
"agency_adhoc",
"agency.adhoc.dotgov.txt"
)
DOMAIN_ADDITIONAL = ( DOMAIN_ADDITIONAL = (
"domain_additional", "domain_additional",
"domainadditionaldatalink.adhoc.dotgov.txt", "domainadditionaldatalink.adhoc.dotgov.txt",
) )
DOMAIN_ESCROW = ( DOMAIN_ESCROW = ("domain_escrow", "escrow_domains.daily.dotgov.GOV.txt")
"domain_escrow", DOMAIN_ADHOC = ("domain_adhoc", "domaintypes.adhoc.dotgov.txt")
"escrow_domains.daily.dotgov.GOV.txt" ORGANIZATION_ADHOC = ("organization_adhoc", "organization.adhoc.dotgov.txt")
) AUTHORITY_ADHOC = ("authority_adhoc", "authority.adhoc.dotgov.txt")
DOMAIN_ADHOC = (
"domain_adhoc",
"domaintypes.adhoc.dotgov.txt"
)
ORGANIZATION_ADHOC = (
"organization_adhoc",
"organization.adhoc.dotgov.txt"
)
AUTHORITY_ADHOC = (
"authority_adhoc",
"authority.adhoc.dotgov.txt"
)

90
src/registrar/management/commands/utility/extra_transition_domain_helper.py
View file

@ -70,9 +70,14 @@ class FileTransitionLog:
dict_name = (file_type, domain_name) dict_name = (file_type, domain_name)
self._add_to_log_list(dict_name, log) self._add_to_log_list(dict_name, log)
def create_log_item( def create_log_item(
self, file_type, code, message, domain_name=None, add_to_list=True, minimal_logging=True self,
file_type,
code,
message,
domain_name=None,
add_to_list=True,
minimal_logging=True,
): ):
"""Creates and returns an LogItem object. """Creates and returns an LogItem object.
@ -107,9 +112,7 @@ class FileTransitionLog:
for parent_log in self.logs: for parent_log in self.logs:
for child_log in parent_log: for child_log in parent_log:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
True, True, child_log.message, child_log.severity
child_log.message,
child_log.severity
) )
def display_logs_by_domain_name(self, domain_name, restrict_type=LogCode.DEFAULT): def display_logs_by_domain_name(self, domain_name, restrict_type=LogCode.DEFAULT):
@ -128,9 +131,7 @@ class FileTransitionLog:
for log in domain_logs: for log in domain_logs:
TerminalHelper.print_conditional( TerminalHelper.print_conditional(
restrict_type != log.code, restrict_type != log.code, log.message, log.code
log.message,
log.code
) )
def get_logs(self, file_type, domain_name): def get_logs(self, file_type, domain_name):
@ -213,19 +214,20 @@ class LoadExtraTransitionDomain:
logger.info( logger.info(
f"""{TerminalColors.OKGREEN} f"""{TerminalColors.OKGREEN}
============= FINISHED =============== ============= FINISHED ===============
Updated {len(updated_transition_domains)} transition domain entries: Updated {len(updated_transition_domains)} transition domain entries
{[domain for domain in updated_transition_domains]}
{TerminalColors.ENDC} {TerminalColors.ENDC}
""" """
) )
else: else:
# TODO - update # TODO - update
TerminalHelper.print_conditional(self.debug, f"{TerminalHelper.array_as_string(updated_transition_domains)}") TerminalHelper.print_conditional(
self.debug,
f"{TerminalHelper.array_as_string(updated_transition_domains)}",
)
logger.error( logger.error(
f"""{TerminalColors.FAIL} f"""{TerminalColors.FAIL}
============= FINISHED WITH ERRORS =============== ============= FINISHED WITH ERRORS ===============
Updated {len(updated_transition_domains)} transition domain entries: Updated {len(updated_transition_domains)} transition domain entries,
{[domain for domain in updated_transition_domains]}
Failed to update {failed_count} transition domain entries: Failed to update {failed_count} transition domain entries:
{[domain for domain in failed_transition_domains]} {[domain for domain in failed_transition_domains]}
{TerminalColors.ENDC} {TerminalColors.ENDC}
@ -237,7 +239,8 @@ class LoadExtraTransitionDomain:
total_transition_domains = len(updated_transition_domains) total_transition_domains = len(updated_transition_domains)
total_updates_made = TransitionDomain.objects.all().count() total_updates_made = TransitionDomain.objects.all().count()
if total_transition_domains != total_updates_made: if total_transition_domains != total_updates_made:
logger.error(f"""{TerminalColors.FAIL} logger.error(
f"""{TerminalColors.FAIL}
WARNING: something went wrong processing domain information data. WARNING: something went wrong processing domain information data.
Total Transition Domains expecting a data update: {total_transition_domains} Total Transition Domains expecting a data update: {total_transition_domains}
@ -248,7 +251,8 @@ class LoadExtraTransitionDomain:
corrupt data. Please check logs to diagnose. corrupt data. Please check logs to diagnose.
----- TERMINATING ---- ----- TERMINATING ----
""") """
)
sys.exit() sys.exit()
def parse_creation_expiration_data(self, domain_name, transition_domain): def parse_creation_expiration_data(self, domain_name, transition_domain):
@ -265,16 +269,12 @@ class LoadExtraTransitionDomain:
"Could not add epp_creation_date and epp_expiration_date " "Could not add epp_creation_date and epp_expiration_date "
f"on {domain_name}, no data exists.", f"on {domain_name}, no data exists.",
domain_name, domain_name,
not self.debug not self.debug,
) )
return transition_domain return transition_domain
creation_exists = ( creation_exists = transition_domain.epp_creation_date is not None
transition_domain.epp_creation_date is not None expiration_exists = transition_domain.epp_expiration_date is not None
)
expiration_exists = (
transition_domain.epp_expiration_date is not None
)
transition_domain.epp_creation_date = info.creationdate transition_domain.epp_creation_date = info.creationdate
transition_domain.epp_expiration_date = info.expirationdate transition_domain.epp_expiration_date = info.expirationdate
@ -311,7 +311,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR, LogCode.ERROR,
f"Could not add federal_agency on {domain_name}, no data exists.", f"Could not add federal_agency on {domain_name}, no data exists.",
domain_name, domain_name,
not self.debug not self.debug,
) )
return transition_domain return transition_domain
@ -326,7 +326,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR, LogCode.ERROR,
f"Could not add inactive agency {info.agencyname} on {domain_name}", f"Could not add inactive agency {info.agencyname} on {domain_name}",
domain_name, domain_name,
not self.debug not self.debug,
) )
return transition_domain return transition_domain
@ -336,7 +336,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR, LogCode.ERROR,
f"Could not add non-federal agency {info.agencyname} on {domain_name}", f"Could not add non-federal agency {info.agencyname} on {domain_name}",
domain_name, domain_name,
not self.debug not self.debug,
) )
return transition_domain return transition_domain
@ -369,7 +369,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR, LogCode.ERROR,
f"Could not add domain_type on {domain_name}, no data exists.", f"Could not add domain_type on {domain_name}, no data exists.",
domain_name, domain_name,
not self.debug not self.debug,
) )
return transition_domain return transition_domain
@ -392,7 +392,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR, LogCode.ERROR,
f"Could not add inactive domain_type {domain_type[0]} on {domain_name}", f"Could not add inactive domain_type {domain_type[0]} on {domain_name}",
domain_name, domain_name,
not self.debug not self.debug,
) )
return transition_domain return transition_domain
@ -453,7 +453,7 @@ class LoadExtraTransitionDomain:
LogCode.ERROR, LogCode.ERROR,
f"Could not add organization_name on {domain_name}, no data exists.", f"Could not add organization_name on {domain_name}, no data exists.",
domain_name, domain_name,
not self.debug not self.debug,
) )
return transition_domain return transition_domain
@ -487,7 +487,7 @@ class LoadExtraTransitionDomain:
LogCode.INFO, LogCode.INFO,
f"Added {var_name} as '{changed_value}' on {domain_name}", f"Added {var_name} as '{changed_value}' on {domain_name}",
domain_name, domain_name,
not self.debug not self.debug,
) )
else: else:
self.parse_logs.create_log_item( self.parse_logs.create_log_item(
@ -495,7 +495,7 @@ class LoadExtraTransitionDomain:
LogCode.WARNING, LogCode.WARNING,
f"Updated existing {var_name} to '{changed_value}' on {domain_name}", f"Updated existing {var_name} to '{changed_value}' on {domain_name}",
domain_name, domain_name,
not self.debug not self.debug,
) )
# Property getters, i.e. orgid or domaintypeid # Property getters, i.e. orgid or domaintypeid
@ -615,7 +615,9 @@ class LoadExtraTransitionDomain:
desired_type = self.parsed_data_container.file_data.get(file_type) desired_type = self.parsed_data_container.file_data.get(file_type)
if desired_type is None: if desired_type is None:
self.parse_logs.create_log_item( self.parse_logs.create_log_item(
file_type, LogCode.ERROR, f"Type {file_type} does not exist", file_type,
LogCode.ERROR,
f"Type {file_type} does not exist",
) )
return None return None
@ -624,10 +626,13 @@ class LoadExtraTransitionDomain:
obj = desired_type.data.get(desired_id) obj = desired_type.data.get(desired_id)
if obj is None: if obj is None:
self.parse_logs.create_log_item( self.parse_logs.create_log_item(
file_type, LogCode.ERROR, f"Id {desired_id} does not exist for {file_type.value[0]}" file_type,
LogCode.ERROR,
f"Id {desired_id} does not exist for {file_type.value[0]}",
) )
return obj return obj
# TODO - change name # TODO - change name
@dataclass @dataclass
class FileDataHolder: class FileDataHolder:
@ -898,26 +903,15 @@ class ExtraTransitionDomain:
file_type.data = {} file_type.data = {}
def parse_csv_file( def parse_csv_file(
self, self, file, seperator, dataclass_type, id_field, is_domain_escrow=False
file,
seperator,
dataclass_type,
id_field,
is_domain_escrow=False
): ):
# Domain escrow is an edge case # Domain escrow is an edge case
if is_domain_escrow: if is_domain_escrow:
item_to_return = self._read_domain_escrow( item_to_return = self._read_domain_escrow(file, seperator)
file,
seperator
)
return item_to_return return item_to_return
else: else:
item_to_return = self._read_csv_file( item_to_return = self._read_csv_file(
file, file, seperator, dataclass_type, id_field
seperator,
dataclass_type,
id_field
) )
return item_to_return return item_to_return
@ -953,7 +947,9 @@ class ExtraTransitionDomain:
f"Found bad data in {file}. Attempting to clean." f"Found bad data in {file}. Attempting to clean."
f"{TerminalColors.ENDC}" f"{TerminalColors.ENDC}"
) )
updated_file_content = self.replace_bad_seperators(file, f"{seperator}", ";badseperator;") updated_file_content = self.replace_bad_seperators(
file, f"{seperator}", ";badseperator;"
)
dict_data = {} dict_data = {}
break break

52
src/registrar/management/commands/utility/terminal_helper.py
View file

@ -4,6 +4,7 @@ import sys
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class LogCode(Enum): class LogCode(Enum):
"""Stores the desired log severity """Stores the desired log severity
@ -21,6 +22,7 @@ class LogCode(Enum):
DEBUG = 4 DEBUG = 4
DEFAULT = 5 DEFAULT = 5
class TerminalColors: class TerminalColors:
"""Colors for terminal outputs """Colors for terminal outputs
(makes reading the logs WAY easier)""" (makes reading the logs WAY easier)"""
@ -81,7 +83,14 @@ class TerminalHelper:
The "answer" return value is True for "yes" or False for "no". The "answer" return value is True for "yes" or False for "no".
""" """
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False, "e": "exit"} valid = {
"yes": True,
"y": True,
"ye": True,
"no": False,
"n": False,
"e": "exit",
}
if default is None: if default is None:
prompt = " [y/n] " prompt = " [y/n] "
elif default == "yes": elif default == "yes":
@ -105,16 +114,14 @@ class TerminalHelper:
# @staticmethod # @staticmethod
def array_as_string(array_to_convert: []) -> str: def array_as_string(array_to_convert: []) -> str:
array_as_string = "{}".format( array_as_string = "{}".format("\n".join(map(str, array_to_convert)))
"\n".join(map(str, array_to_convert))
)
return array_as_string return array_as_string
@staticmethod @staticmethod
def print_conditional( def print_conditional(
print_condition: bool, print_condition: bool,
print_statement: str, print_statement: str,
log_severity: LogCode = LogCode.DEFAULT log_severity: LogCode = LogCode.DEFAULT,
): ):
"""This function reduces complexity of debug statements """This function reduces complexity of debug statements
in other functions. in other functions.
@ -181,29 +188,40 @@ class TerminalHelper:
@staticmethod @staticmethod
def get_file_line_count(filepath: str) -> int: def get_file_line_count(filepath: str) -> int:
with open(filepath,'r') as file: with open(filepath, "r") as file:
li = file.readlines() li = file.readlines()
total_line = len(li) total_line = len(li)
return total_line return total_line
@staticmethod @staticmethod
def print_to_file_conditional(print_condition: bool, filename: str, file_directory: str, file_contents: str): def print_to_file_conditional(
"""Sometimes logger outputs get insanely huge. print_condition: bool, filename: str, file_directory: str, file_contents: str
""" ):
if (print_condition): """Sometimes logger outputs get insanely huge."""
if print_condition:
# Add a slash if the last character isn't one # Add a slash if the last character isn't one
if file_directory and file_directory[-1] != "/": if file_directory and file_directory[-1] != "/":
file_directory += "/" file_directory += "/"
# Assemble filepath # Assemble filepath
filepath = f"{file_directory}{filename}.txt" filepath = f"{file_directory}{filename}.txt"
# Write to file # Write to file
logger.info(f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}") logger.info(
f"{TerminalColors.MAGENTA}Writing to file {filepath}...{TerminalColors.ENDC}"
)
with open(f"{filepath}", "w+") as f: with open(f"{filepath}", "w+") as f:
f.write(file_contents) f.write(file_contents)
@staticmethod @staticmethod
def printProgressBar (iteration, total, prefix = 'Progress:', suffix = 'Complete', decimals = 1, length = 100, fill = '█', printEnd = "\r"): def printProgressBar(
iteration,
total,
prefix="Progress:",
suffix="Complete",
decimals=1,
length=100,
fill="█",
printEnd="\r",
):
""" """
Call in a loop to create terminal progress bar Call in a loop to create terminal progress bar
@params: @params:
@ -227,10 +245,12 @@ class TerminalHelper:
printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50) printProgressBar(i + 1, l, prefix = 'Progress:', suffix = 'Complete', length = 50)
""" """
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) percent = ("{0:." + str(decimals) + "f}").format(
100 * (iteration / float(total))
)
filledLength = int(length * iteration // total) filledLength = int(length * iteration // total)
bar = fill * filledLength + '-' * (length - filledLength) bar = fill * filledLength + "-" * (length - filledLength)
print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd) print(f"\r{prefix} |{bar}| {percent}% {suffix}", end=printEnd)
# Print New Line on Complete # Print New Line on Complete
if iteration == total: if iteration == total:
print() print()

25
src/registrar/management/commands/utility/transition_domain_arguments.py
View file

@ -3,6 +3,7 @@ from typing import Optional
from registrar.management.commands.utility.epp_data_containers import EnumFilenames from registrar.management.commands.utility.epp_data_containers import EnumFilenames
@dataclass @dataclass
class TransitionDomainArguments: class TransitionDomainArguments:
"""Stores arguments for load_transition_domain, structurally a mix """Stores arguments for load_transition_domain, structurally a mix
@ -36,14 +37,26 @@ class TransitionDomainArguments:
# Filenames # # Filenames #
## Adhocs ## ## Adhocs ##
agency_adhoc_filename: Optional[str] = field(default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True) agency_adhoc_filename: Optional[str] = field(
domain_adhoc_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True) default=EnumFilenames.AGENCY_ADHOC.value[1], repr=True
organization_adhoc_filename: Optional[str] = field(default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True) )
authority_adhoc_filename: Optional[str] = field(default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True) domain_adhoc_filename: Optional[str] = field(
default=EnumFilenames.DOMAIN_ADHOC.value[1], repr=True
)
organization_adhoc_filename: Optional[str] = field(
default=EnumFilenames.ORGANIZATION_ADHOC.value[1], repr=True
)
authority_adhoc_filename: Optional[str] = field(
default=EnumFilenames.AUTHORITY_ADHOC.value[1], repr=True
)
## Data files ## ## Data files ##
domain_escrow_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True) domain_escrow_filename: Optional[str] = field(
domain_additional_filename: Optional[str] = field(default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True) default=EnumFilenames.DOMAIN_ESCROW.value[1], repr=True
)
domain_additional_filename: Optional[str] = field(
default=EnumFilenames.DOMAIN_ADDITIONAL.value[1], repr=True
)
domain_contacts_filename: Optional[str] = field(default=None, repr=True) domain_contacts_filename: Optional[str] = field(default=None, repr=True)
domain_statuses_filename: Optional[str] = field(default=None, repr=True) domain_statuses_filename: Optional[str] = field(default=None, repr=True)
contacts_filename: Optional[str] = field(default=None, repr=True) contacts_filename: Optional[str] = field(default=None, repr=True)

31
src/registrar/tests/test_transition_domain_migrations.py
View file

@ -13,6 +13,7 @@ from registrar.models import (
from django.core.management import call_command from django.core.management import call_command
from unittest.mock import patch from unittest.mock import patch
class TestMigrations(TestCase): class TestMigrations(TestCase):
def setUp(self): def setUp(self):
""" """ """ """
@ -46,24 +47,30 @@ class TestMigrations(TestCase):
UserDomainRole.objects.all().delete() UserDomainRole.objects.all().delete()
def run_load_domains(self): def run_load_domains(self):
with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): with patch(
"registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit",
return_value=True,
):
call_command( call_command(
"load_transition_domain", "load_transition_domain",
self.migration_json_filename, self.migration_json_filename,
directory=self.test_data_file_location directory=self.test_data_file_location,
) )
def run_transfer_domains(self): def run_transfer_domains(self):
call_command("transfer_transition_domains_to_domains") call_command("transfer_transition_domains_to_domains")
def run_master_script(self): def run_master_script(self):
with patch('registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit', return_value=True): with patch(
"registrar.management.commands.utility.terminal_helper.TerminalHelper.query_yes_no_exit",
return_value=True,
):
call_command( call_command(
"master_domain_migrations", "master_domain_migrations",
runMigrations=True, runMigrations=True,
migrationDirectory=self.test_data_file_location, migrationDirectory=self.test_data_file_location,
migrationJSON=self.migration_json_filename, migrationJSON=self.migration_json_filename,
disablePrompts=True disablePrompts=True,
) )
def compare_tables( def compare_tables(
@ -242,7 +249,7 @@ class TestMigrations(TestCase):
federal_type="Executive", federal_type="Executive",
federal_agency="InnoZ", federal_agency="InnoZ",
epp_creation_date=None, epp_creation_date=None,
epp_expiration_date=None epp_expiration_date=None,
), ),
TransitionDomain( TransitionDomain(
username="reginald.ratcliff4@test.com", username="reginald.ratcliff4@test.com",
@ -254,20 +261,21 @@ class TestMigrations(TestCase):
federal_type=None, federal_type=None,
federal_agency=None, federal_agency=None,
epp_creation_date=None, epp_creation_date=None,
epp_expiration_date=None epp_expiration_date=None,
) ),
] ]
expected_transition_domains = TransitionDomain.objects.filter(username="alexandra.bobbitt5@test.com") expected_transition_domains = TransitionDomain.objects.filter(
username="alexandra.bobbitt5@test.com"
)
self.assertEqual(expected_transition_domains.count(), 1) self.assertEqual(expected_transition_domains.count(), 1)
expected_transition_domain = expected_transition_domains.get() expected_transition_domain = expected_transition_domains.get()
#TransitionDomain.objects.filter(domain_name = "fakewebsite3.gov") # TransitionDomain.objects.filter(domain_name = "fakewebsite3.gov")
# Afterwards, their values should be what we expect # Afterwards, their values should be what we expect
all_transition_domains = TransitionDomain.objects.all() all_transition_domains = TransitionDomain.objects.all()
for domain in all_transition_domains: for domain in all_transition_domains:
for expected in expected_transition_domains: for expected in expected_transition_domains:
# This data gets created when the object is, # This data gets created when the object is,
# so we should just match it. Not relevant # so we should just match it. Not relevant
# to the added data. # to the added data.
@ -323,7 +331,7 @@ class TestMigrations(TestCase):
testdomain = testdomain_domains.get() testdomain = testdomain_domains.get()
self.assertEqual(testdomain.expiration_date, datetime.date(2023, 9, 30)) self.assertEqual(testdomain.expiration_date, datetime.date(2023, 9, 30))
#self.assertEqual(testdomain.created_at, "test") # self.assertEqual(testdomain.created_at, "test")
self.assertEqual(testdomain.name, "fakewebsite2.gov") self.assertEqual(testdomain.name, "fakewebsite2.gov")
self.assertEqual(testdomain.state, "on hold") self.assertEqual(testdomain.state, "on hold")
@ -386,7 +394,6 @@ class TestMigrations(TestCase):
self.assertEqual(Users.count(), 1) self.assertEqual(Users.count(), 1)
self.assertEqual(anomaly.creator, Users.get()) self.assertEqual(anomaly.creator, Users.get())
def test_transfer_transition_domains_to_domains(self): def test_transfer_transition_domains_to_domains(self):
self.run_load_domains() self.run_load_domains()
self.run_transfer_domains() self.run_transfer_domains()