zydronium/manage.get.gov
1
0
Fork 0
mirror of https://github.com/cisagov/manage.get.gov.git synced 2025-05-30 01:10:04 +02:00
Code Issues Projects Releases Packages Wiki Activity

linted

Browse source
This commit is contained in:
CocoByte 2023-09-25 22:44:52 -06:00
parent 36ac4b9046
commit d38fd15520
No known key found for this signature in database
GPG key ID: BBFAA2526384C97F
1 changed files with 190 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

331
src/registrar/management/commands/load_transition_domain.py
View file

@ -96,7 +96,7 @@ class Command(BaseCommand):
action=argparse.BooleanOptionalAction, action=argparse.BooleanOptionalAction,
) )
def handle( def handle( # noqa: C901
self, self,
domain_contacts_filename, domain_contacts_filename,
contacts_filename, contacts_filename,
@ -109,95 +109,66 @@ class Command(BaseCommand):
if options.get("resetTable"): if options.get("resetTable"):
confirmReset = query_yes_no( confirmReset = query_yes_no(
f""" f"""
{termColors.FAIL} {termColors.FAIL}
WARNING: Resetting the table will permanently delete all the data! WARNING: Resetting the table will permanently delete all
Are you sure you want to continue?{termColors.ENDC}""" the data!
Are you sure you want to continue?{termColors.ENDC}"""
) )
if confirmReset: if confirmReset:
logger.info( logger.info(
f"""{termColors.WARNING} f"""{termColors.WARNING}
----------Clearing Table Data---------- ----------Clearing Table Data----------
(please wait) (please wait)
{termColors.ENDC}""" {termColors.ENDC}"""
) )
TransitionDomain.objects.all().delete() TransitionDomain.objects.all().delete()
debugOn = options.get("debug") debug_on = options.get("debug")
debug_MaxEntriesToParse = int( debug_max_entries_to_parse = int(
options.get("limitParse") options.get("limitParse")
) # set to 0 to parse all entries ) # set to 0 to parse all entries
if debugOn: self.print_debug_mode_statements(debug_on, debug_max_entries_to_parse)
logger.info(
f"""{termColors.OKCYAN}
----------DEBUG MODE ON----------
Detailed print statements activated.
{termColors.ENDC}
"""
)
if debug_MaxEntriesToParse > 0:
logger.info(
f"""{termColors.OKCYAN}
----------LIMITER ON----------
Parsing of entries will be limited to {debug_MaxEntriesToParse} lines per file.")
Detailed print statements activated.
{termColors.ENDC}
"""
)
# STEP 1: # STEP 1:
# Create mapping of domain name -> status # Create mapping of domain name -> status
# TODO: figure out latest status domain_status_dictionary = self.get_domain_user_dict(
domain_status_dictionary = defaultdict( domain_statuses_filename, sep
str )
) # NOTE: how to determine "most recent" status?
logger.info("Reading domain statuses data file %s", domain_statuses_filename)
with open(domain_statuses_filename, "r") as domain_statuses_file:
for row in csv.reader(domain_statuses_file, delimiter=sep):
domainName = row[0].lower()
domainStatus = row[1].lower()
# print("adding "+domainName+", "+domainStatus)
domain_status_dictionary[domainName] = domainStatus
logger.info("Loaded statuses for %d domains", len(domain_status_dictionary))
# STEP 2: # STEP 2:
# Create mapping of userId -> emails NOTE: is this one to many?? # Create mapping of userId -> email
user_emails_dictionary = defaultdict(list) # each contact has a list of e-mails user_emails_dictionary = self.get_user_emails_dict(contacts_filename, sep)
logger.info("Reading domain-contacts data file %s", domain_contacts_filename)
with open(contacts_filename, "r") as contacts_file:
for row in csv.reader(contacts_file, delimiter=sep):
userId = row[0]
user_email = row[6]
user_emails_dictionary[userId].append(user_email)
logger.info("Loaded emails for %d users", len(user_emails_dictionary))
# STEP 3: # STEP 3:
# TODO: Need to add logic for conflicting domain status entries # TODO: Need to add logic for conflicting domain status
# entries
# (which should not exist, but might) # (which should not exist, but might)
# TODO: log statuses found that don't map to the ones we have (count occurences) # TODO: log statuses found that don't map to the ones
# we have (count occurences)
to_create = [] to_create = []
# keep track of statuses that don't match our available status values # keep track of statuses that don't match our available
# status values
outlier_statuses = [] outlier_statuses = []
total_outlier_statuses = 0
# keep track of domains that have no known status # keep track of domains that have no known status
domains_without_status = [] domains_without_status = []
total_domains_without_status = 0
# keep track of users that have no e-mails # keep track of users that have no e-mails
users_without_email = [] users_without_email = []
total_users_without_email = 0
# keep track of dupliucations.. # keep track of duplications..
duplicate_domains = [] duplicate_domains = []
duplicate_domain_user_combos = [] duplicate_domain_user_combos = []
# keep track of domains we UPDATED (instead of ones we added) # keep track of domains we ADD or UPDATE
total_updated_domain_entries = 0 total_updated_domain_entries = 0
total_new_entries = 0 total_new_entries = 0
# if we are limiting our parse (for testing purposes, keep
# track of total rows parsed)
total_rows_parsed = 0 total_rows_parsed = 0
logger.info("Reading domain-contacts data file %s", domain_contacts_filename) logger.info("Reading domain-contacts data file %s", domain_contacts_filename)
@ -207,124 +178,114 @@ class Command(BaseCommand):
# fields are just domain, userid, role # fields are just domain, userid, role
# lowercase the domain names # lowercase the domain names
domainName = row[0].lower() new_entry_domainName = row[0].lower()
userId = row[1] userId = row[1]
domainStatus = TransitionDomain.StatusChoices.CREATED new_entry_status = TransitionDomain.StatusChoices.CREATED
userEmail = "" new_entry_email = ""
emailSent = False new_entry_emailSent = False
# TODO: how to know if e-mail was sent? # TODO: how to know if e-mail was sent?
if domainName not in domain_status_dictionary: if new_entry_domainName not in domain_status_dictionary:
# this domain has no status...default to "Create" # this domain has no status...default to "Create"
if domainName not in domains_without_status: if new_entry_domainName not in domains_without_status:
domains_without_status.append(domainName) domains_without_status.append(new_entry_domainName)
total_domains_without_status += 1
else: else:
originalStatus = domain_status_dictionary[domainName] original_status = domain_status_dictionary[new_entry_domainName]
# print(originalStatus) # print(originalStatus)
if originalStatus in TransitionDomain.StatusChoices.values: mapped_status = self.get_mapped_status(original_status)
# This status maps directly to our available status options if mapped_status is None:
domainStatus = originalStatus logger.info("Unknown status: " + original_status)
outlier_statuses.append(original_status)
else: else:
# Map all other status as follows; new_entry_status = mapped_status
# "serverHold” fields will map to hold clientHold to hold
# and any ok state should map to Ready.
# Check if there are any statuses that are not
# serverhold, client hold or OK in the original data set.
if (
originalStatus.lower() == "serverhold"
or originalStatus.lower() == "clienthold"
):
domainStatus = TransitionDomain.StatusChoices.HOLD
elif originalStatus.lower() != "ok":
if originalStatus not in outlier_statuses:
outlier_statuses.append(originalStatus)
logger.info("Unknown status: " + originalStatus)
total_outlier_statuses += 1
if userId not in user_emails_dictionary: if userId not in user_emails_dictionary:
# this user has no e-mail...this should never happen # this user has no e-mail...this should never happen
# users_without_email.add(userId) if userId not in users_without_email:
# print("no e-mail found for user: "+userId) users_without_email.append(userId)
total_users_without_email += 1
else: else:
userEmail = user_emails_dictionary[userId] new_entry_email = user_emails_dictionary[userId]
# Check for duplicate data in the file we are parsing so we do not add duplicates # Check for duplicate data in the file we are
# NOTE: Currently, we allow duplicate domains, but not duplicate domain-user pairs. # parsing so we do not add duplicates
# However, track duplicate domains for now, since we are still deciding on whether # NOTE: Currently, we allow duplicate domains,
# but not duplicate domain-user pairs.
# However, track duplicate domains for now,
# since we are still deciding on whether
# to make this field unique or not. ~10/25/2023 # to make this field unique or not. ~10/25/2023
tempEntry_domain = next( tempEntry_domain = next(
(x for x in to_create if x.domain_name == domainName), None (x for x in to_create if x.domain_name == new_entry_domainName),
None,
) )
tempEntry_domainUserPair = next( tempEntry_domainUserPair = next(
( (
x x
for x in to_create for x in to_create
if x.username == userEmail and x.domain_name == domainName if x.username == new_entry_email
and x.domain_name == new_entry_domainName
), ),
None, None,
) )
if tempEntry_domain is not None: if tempEntry_domain is not None:
if debugOn: if debug_on:
logger.info( logger.info(
f"{termColors.WARNING} DUPLICATE Verisign entries found for domain: {domainName} {termColors.ENDC}" f"{termColors.WARNING} DUPLICATE Verisign entries found for domain: {new_entry_domainName} {termColors.ENDC}" # noqa
) )
if domainName not in duplicate_domains: if new_entry_domainName not in duplicate_domains:
duplicate_domains.append(domainName) duplicate_domains.append(new_entry_domainName)
if tempEntry_domainUserPair != None: if tempEntry_domainUserPair is not None:
if debugOn: if debug_on:
logger.info( logger.info(
f""" f"""{termColors.WARNING} DUPLICATE Verisign entries found for domain - user {termColors.BackgroundLightYellow} PAIR {termColors.ENDC}{termColors.WARNING}:
{termColors.WARNING} {new_entry_domainName} - {new_entry_email} {termColors.ENDC}""" # noqa
DUPLICATE Verisign entries found for domain - user {termColors.BackgroundLightYellow} PAIR {termColors.ENDC}{termColors.WARNING}:
{domainName} - {user_email} {termColors.ENDC}"""
) )
if tempEntry_domainUserPair not in duplicate_domain_user_combos: if tempEntry_domainUserPair not in duplicate_domain_user_combos:
duplicate_domain_user_combos.append(tempEntry_domainUserPair) duplicate_domain_user_combos.append(tempEntry_domainUserPair)
else: else:
try: try:
existingEntry = TransitionDomain.objects.get( existingEntry = TransitionDomain.objects.get(
username=userEmail, domain_name=domainName username=new_entry_email, domain_name=new_entry_domainName
) )
if existingEntry.status != domainStatus: if existingEntry.status != new_entry_status:
# DEBUG: # DEBUG:
if debugOn: if debug_on:
logger.info( logger.info(
f"""{termColors.OKCYAN} f"""{termColors.OKCYAN}
Updating entry: {existingEntry} Updating entry: {existingEntry}
Status: {existingEntry.status} > {domainStatus} Status: {existingEntry.status} > {new_entry_status}
Email Sent: {existingEntry.email_sent} > {emailSent} Email Sent: {existingEntry.email_sent} > {new_entry_emailSent}
{termColors.ENDC}""" {termColors.ENDC}"""
) )
existingEntry.status = domainStatus existingEntry.status = new_entry_status
existingEntry.email_sent = emailSent existingEntry.email_sent = new_entry_emailSent
existingEntry.save() existingEntry.save()
except TransitionDomain.DoesNotExist: except TransitionDomain.DoesNotExist:
# no matching entry, make one # no matching entry, make one
newEntry = TransitionDomain( newEntry = TransitionDomain(
username=userEmail, username=new_entry_email,
domain_name=domainName, domain_name=new_entry_domainName,
status=domainStatus, status=new_entry_status,
email_sent=emailSent, email_sent=new_entry_emailSent,
) )
to_create.append(newEntry) to_create.append(newEntry)
total_new_entries += 1 total_new_entries += 1
# DEBUG: # DEBUG:
if debugOn: if debug_on:
logger.info( logger.info(
f"{termColors.OKCYAN} Adding entry {total_new_entries}: {newEntry} {termColors.ENDC}" f"{termColors.OKCYAN} Adding entry {total_new_entries}: {newEntry} {termColors.ENDC}" # noqa
) )
except TransitionDomain.MultipleObjectsReturned: except TransitionDomain.MultipleObjectsReturned:
logger.info( logger.info(
f""" f"""
{termColors.FAIL} {termColors.FAIL}
!!! ERROR: duplicate entries exist in the transtion_domain table for domain: {domainName} !!! ERROR: duplicate entries exist in the
transtion_domain table for domain:
{new_entry_domainName}
----------TERMINATING----------""" ----------TERMINATING----------"""
) )
import sys import sys
@ -332,14 +293,14 @@ DUPLICATE Verisign entries found for domain - user {termColors.BackgroundLightYe
sys.exit() sys.exit()
# DEBUG: # DEBUG:
if debugOn or debug_MaxEntriesToParse > 0: if debug_on or debug_max_entries_to_parse > 0:
if ( if (
total_rows_parsed > debug_MaxEntriesToParse total_rows_parsed > debug_max_entries_to_parse
and debug_MaxEntriesToParse != 0 and debug_max_entries_to_parse != 0
): ):
logger.info( logger.info(
f"""{termColors.WARNING} f"""{termColors.WARNING}
----BREAK---- ----PARSE LIMIT REACHED. HALTING PARSER.----
{termColors.ENDC} {termColors.ENDC}
""" """
) )
@ -351,15 +312,88 @@ DUPLICATE Verisign entries found for domain - user {termColors.BackgroundLightYe
f"""{termColors.OKGREEN} f"""{termColors.OKGREEN}
============= FINISHED =============== ============= FINISHED ===============
Created {total_new_entries} transition domain entries, Created {total_new_entries} transition domain entries,
updated {total_updated_domain_entries} transition domain entries updated {total_updated_domain_entries} transition domain entries
{termColors.ENDC} {termColors.ENDC}
""" """
) )
# Print a summary of findings (duplicate entries, missing data..etc.) # Print a summary of findings (duplicate entries,
# missing data..etc.)
self.print_summary_duplications(
duplicate_domain_user_combos, duplicate_domains, users_without_email
)
self.print_summary_status_findings(domains_without_status, outlier_statuses)
def print_debug_mode_statements(self, debug_on, debug_max_entries_to_parse):
if debug_on:
logger.info(
f"""{termColors.OKCYAN}
----------DEBUG MODE ON----------
Detailed print statements activated.
{termColors.ENDC}
"""
)
if debug_max_entries_to_parse > 0:
logger.info(
f"""{termColors.OKCYAN}
----------LIMITER ON----------
Parsing of entries will be limited to
{debug_max_entries_to_parse} lines per file.")
Detailed print statements activated.
{termColors.ENDC}
"""
)
def get_domain_user_dict(self, domain_statuses_filename, sep):
"""Creates a mapping of domain name -> status"""
# TODO: figure out latest status
domain_status_dictionary = defaultdict(str)
# NOTE: how to determine "most recent" status?
logger.info("Reading domain statuses data file %s", domain_statuses_filename)
with open(domain_statuses_filename, "r") as domain_statuses_file: # noqa
for row in csv.reader(domain_statuses_file, delimiter=sep):
domainName = row[0].lower()
domainStatus = row[1].lower()
# print("adding "+domainName+", "+domainStatus)
domain_status_dictionary[domainName] = domainStatus
logger.info("Loaded statuses for %d domains", len(domain_status_dictionary))
return domain_status_dictionary
def get_user_emails_dict(self, contacts_filename, sep):
"""Creates mapping of userId -> emails"""
# NOTE: is this one to many??
user_emails_dictionary = defaultdict(list)
logger.info("Reading domain-contacts data file %s", contacts_filename)
with open(contacts_filename, "r") as contacts_file:
for row in csv.reader(contacts_file, delimiter=sep):
userId = row[0]
user_email = row[6]
user_emails_dictionary[userId].append(user_email)
logger.info("Loaded emails for %d users", len(user_emails_dictionary))
return user_emails_dictionary
def get_mapped_status(self, status_to_map):
# Map statuses as follows;
# "serverHold” fields will map to hold clientHold to hold
# and any ok state should map to Ready.
# Check if there are any statuses that are not
# serverhold, client hold or OK in the original data set.
status_maps = {
"hold": TransitionDomain.StatusChoices.HOLD,
"serverhold": TransitionDomain.StatusChoices.HOLD,
"clienthold": TransitionDomain.StatusChoices.HOLD,
"created": TransitionDomain.StatusChoices.CREATED,
"ok": TransitionDomain.StatusChoices.CREATED,
}
return status_maps[status_to_map]
def print_summary_duplications(
self, duplicate_domain_user_combos, duplicate_domains, users_without_email
):
totalDupDomainUserPairs = len(duplicate_domain_user_combos) totalDupDomainUserPairs = len(duplicate_domain_user_combos)
totalDupDomains = len(duplicate_domains) totalDupDomains = len(duplicate_domains)
total_users_without_email = len(users_without_email)
if total_users_without_email > 0: if total_users_without_email > 0:
logger.warning( logger.warning(
"No e-mails found for users: {}".format( "No e-mails found for users: {}".format(
@ -375,40 +409,55 @@ DUPLICATE Verisign entries found for domain - user {termColors.BackgroundLightYe
) )
logger.warning( logger.warning(
f"""{termColors.WARNING} f"""{termColors.WARNING}
----DUPLICATES FOUND-----
{totalDupDomainUserPairs} DOMAIN - USER pairs were NOT unique in the supplied data files; ----DUPLICATES FOUND-----
{temp_dupPairsAsString} {totalDupDomainUserPairs} DOMAIN - USER pairs
were NOT unique in the supplied data files;
{totalDupDomains} DOMAINS were NOT unique in the supplied data files; {temp_dupPairsAsString}
{temp_dupDomainsAsString} {totalDupDomains} DOMAINS were NOT unique in
{termColors.ENDC}""" the supplied data files;
{temp_dupDomainsAsString}
{termColors.ENDC}"""
) )
def print_summary_status_findings(self, domains_without_status, outlier_statuses):
total_domains_without_status = len(domains_without_status)
total_outlier_statuses = len(outlier_statuses)
if total_domains_without_status > 0: if total_domains_without_status > 0:
temp_arrayToString = "{}".format( temp_arrayToString = "{}".format(
", ".join(map(str, domains_without_status)) ", ".join(map(str, domains_without_status))
) )
logger.warning( logger.warning(
f"""{termColors.WARNING} f"""{termColors.WARNING}
----Found {total_domains_without_status} domains without a status (defaulted to READY)-----
{temp_arrayToString} --------------------------------------------
{termColors.ENDC}""" Found {total_domains_without_status} domains
without a status (defaulted to READY)
---------------------------------------------
{temp_arrayToString}
{termColors.ENDC}"""
) )
if total_outlier_statuses > 0: if total_outlier_statuses > 0:
temp_arrayToString = "{}".format(", ".join(map(str, outlier_statuses))) temp_arrayToString = "{}".format(
", ".join(map(str, outlier_statuses))
) # noqa
logger.warning( logger.warning(
f"""{termColors.WARNING} f"""{termColors.WARNING}
----Found {total_outlier_statuses} unaccounted for statuses-----
No mappings found for the following statuses (defaulted to Ready): --------------------------------------------
Found {total_outlier_statuses} unaccounted
{temp_arrayToString} for statuses-
{termColors.ENDC}""" --------------------------------------------
No mappings found for the following statuses
(defaulted to Ready):
{temp_arrayToString}
{termColors.ENDC}"""
) )