From adc2013c7ca1905084dad4fc85d2f933179abcae Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:07:12 -0600 Subject: [PATCH 1/9] Simplify report and handle many items --- .../commands/email_current_metadata_report.py | 109 ++++++++++-------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index dcaf47b06..1e8faac82 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -9,7 +9,7 @@ from datetime import datetime from django.core.management import BaseCommand from django.conf import settings from registrar.utility import csv_export -from registrar.utility.s3_bucket import S3ClientHelper +from io import StringIO from ...utility.email import send_templated_email @@ -17,89 +17,104 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): + """Emails a encrypted zip file containing a csv of our domains and domain requests""" help = ( "Generates and uploads a domain-metadata.csv file to our S3 bucket " "which is based off of all existing Domains." ) + current_date = datetime.now().strftime("%m%d%Y") + email_to: str def add_arguments(self, parser): """Add our two filename arguments.""" - parser.add_argument("--directory", default="migrationdata", help="Desired directory") parser.add_argument( - "--checkpath", - default=True, - help="Flag that determines if we do a check for os.path.exists. Used for test cases", + "--emailTo", + default=settings.DEFAULT_FROM_EMAIL, + help="Defines where we should email this report", ) def handle(self, **options): """Grabs the directory then creates domain-metadata.csv in that directory""" - file_name = "domain-metadata.csv" - # Ensures a slash is added - directory = os.path.join(options.get("directory"), "") - check_path = options.get("checkpath") + self.email_to = options.get("emailTo") + + # Don't email to DEFAULT_FROM_EMAIL when not prod. + if not settings.IS_PRODUCTION and self.email_to == settings.DEFAULT_FROM_EMAIL: + raise ValueError( + "The --emailTo arg must be specified in non-prod environments, " + "and the arg must not equal the DEFAULT_FROM_EMAIL value (aka: help@get.gov)." + ) logger.info("Generating report...") + zip_filename = f"domain-metadata-{self.current_date}.zip" try: - self.email_current_metadata_report(directory, file_name, check_path) + self.email_current_metadata_report(zip_filename) except Exception as err: # TODO - #1317: Notify operations when auto report generation fails raise err else: - logger.info(f"Success! Created {file_name} and successfully sent out an email!") + logger.info(f"Success! Created {zip_filename} and successfully sent out an email!") - def email_current_metadata_report(self, directory, file_name, check_path): + def email_current_metadata_report(self, zip_filename): """Creates a current-metadata.csv file under the specified directory, then uploads it to a AWS S3 bucket. This is done for resiliency reasons in the event our application goes down and/or the email cannot send -- we'll still be able to grab info from the S3 instance""" - s3_client = S3ClientHelper() - file_path = os.path.join(directory, file_name) + reports = { + "Domain report": { + "report_filename": f"domain-metadata-{self.current_date}.csv", + "report_function": csv_export.export_data_type_to_csv, + }, + "Domain request report": { + "report_filename": f"domain-request-metadata-{self.current_date}.csv", + "report_function": csv_export.DomainRequestExport.export_full_domain_request_report, + }, + } + # Set the password equal to our content in SECRET_ENCRYPT_METADATA. + # For local development, this will be "devpwd" unless otherwise set. + override = settings.SECRET_ENCRYPT_METADATA is None and not settings.IS_PRODUCTION + password = "devpwd" if override else settings.SECRET_ENCRYPT_METADATA - # Generate a file locally for upload - with open(file_path, "w") as file: - csv_export.export_data_type_to_csv(file) - - if check_path and not os.path.exists(file_path): - raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") - - s3_client.upload_file(file_path, file_name) - - # Set zip file name - current_date = datetime.now().strftime("%m%d%Y") - current_filename = f"domain-metadata-{current_date}.zip" - - # Pre-set zip file name - encrypted_metadata_output = current_filename - - # Set context for the subject - current_date_str = datetime.now().strftime("%Y-%m-%d") - - # Encrypt the metadata - encrypted_metadata_in_bytes = self._encrypt_metadata( - s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA) - ) + encrypted_zip_in_bytes = self.get_encrypted_zip(zip_filename, reports, password) # Send the metadata file that is zipped send_templated_email( template_name="emails/metadata_body.txt", subject_template_name="emails/metadata_subject.txt", - to_address=settings.DEFAULT_FROM_EMAIL, - context={"current_date_str": current_date_str}, - attachment_file=encrypted_metadata_in_bytes, + to_address=self.email_to, + context={"current_date_str": datetime.now().strftime("%Y-%m-%d")}, + attachment_file=encrypted_zip_in_bytes, ) - def _encrypt_metadata(self, input_file, output_file, password): + + def get_encrypted_zip(self, zip_filename, reports, password): """Helper function for encrypting the attachment file""" - current_date = datetime.now().strftime("%m%d%Y") - current_filename = f"domain-metadata-{current_date}.csv" + # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities and faster # We could also use compression=pyzipper.ZIP_LZMA if we are looking for smaller file size with pyzipper.AESZipFile( - output_file, "w", compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES + zip_filename, "w", compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES ) as f_out: - f_out.setpassword(password) - f_out.writestr(current_filename, input_file) - with open(output_file, "rb") as file_data: + f_out.setpassword(str.encode(password)) + for report_name, report_value in reports.items(): + report_filename = report_value["report_filename"] + report_function = report_value["report_function"] + + report = self.write_and_return_report(report_function) + f_out.writestr(report_filename, report) + logger.info(f"Generated {report_name}") + + # Get the final report for emailing purposes + with open(zip_filename, "rb") as file_data: attachment_in_bytes = file_data.read() + return attachment_in_bytes + + def write_and_return_report(self, report_function): + """Writes a report to a StringIO object given a report_function and returns the string.""" + report_bytes = StringIO() + report_function(report_bytes) + + # Rewind the buffer to the beginning after writing + report_bytes.seek(0) + return report_bytes.read() From de6ff1d7d1f1115d5bf1a93212e3818aa8da2dae Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:18:33 -0600 Subject: [PATCH 2/9] Streamline --- .../commands/email_current_metadata_report.py | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 1e8faac82..3f38bec31 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -23,7 +23,6 @@ class Command(BaseCommand): "which is based off of all existing Domains." ) current_date = datetime.now().strftime("%m%d%Y") - email_to: str def add_arguments(self, parser): """Add our two filename arguments.""" @@ -35,31 +34,27 @@ class Command(BaseCommand): def handle(self, **options): """Grabs the directory then creates domain-metadata.csv in that directory""" - self.email_to = options.get("emailTo") + zip_filename = f"domain-metadata-{self.current_date}.zip" + email_to = options.get("emailTo") # Don't email to DEFAULT_FROM_EMAIL when not prod. - if not settings.IS_PRODUCTION and self.email_to == settings.DEFAULT_FROM_EMAIL: + if not settings.IS_PRODUCTION and email_to == settings.DEFAULT_FROM_EMAIL: raise ValueError( "The --emailTo arg must be specified in non-prod environments, " "and the arg must not equal the DEFAULT_FROM_EMAIL value (aka: help@get.gov)." ) logger.info("Generating report...") - zip_filename = f"domain-metadata-{self.current_date}.zip" try: - self.email_current_metadata_report(zip_filename) + self.email_current_metadata_report(zip_filename, email_to) except Exception as err: # TODO - #1317: Notify operations when auto report generation fails raise err else: logger.info(f"Success! Created {zip_filename} and successfully sent out an email!") - def email_current_metadata_report(self, zip_filename): - """Creates a current-metadata.csv file under the specified directory, - then uploads it to a AWS S3 bucket. This is done for resiliency - reasons in the event our application goes down and/or the email - cannot send -- we'll still be able to grab info from the S3 - instance""" + def email_current_metadata_report(self, zip_filename, email_to): + """Emails a password protected zip containing domain-metadata and domain-request-metadata""" reports = { "Domain report": { "report_filename": f"domain-metadata-{self.current_date}.csv", @@ -81,7 +76,7 @@ class Command(BaseCommand): send_templated_email( template_name="emails/metadata_body.txt", subject_template_name="emails/metadata_subject.txt", - to_address=self.email_to, + to_address=email_to, context={"current_date_str": datetime.now().strftime("%Y-%m-%d")}, attachment_file=encrypted_zip_in_bytes, ) @@ -96,13 +91,10 @@ class Command(BaseCommand): zip_filename, "w", compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES ) as f_out: f_out.setpassword(str.encode(password)) - for report_name, report_value in reports.items(): - report_filename = report_value["report_filename"] - report_function = report_value["report_function"] - - report = self.write_and_return_report(report_function) - f_out.writestr(report_filename, report) - logger.info(f"Generated {report_name}") + for report_name, report in reports.items(): + logger.info(f"Generating {report_name}") + report = self.write_and_return_report(report["report_function"]) + f_out.writestr(report["report_filename"], report) # Get the final report for emailing purposes with open(zip_filename, "rb") as file_data: From a64ecbd0d20f82d5136bd30f905ef9caedcdecdf Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 17 Jun 2024 14:22:10 -0600 Subject: [PATCH 3/9] Lint --- .../management/commands/email_current_metadata_report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 3f38bec31..b76e77608 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -18,6 +18,7 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): """Emails a encrypted zip file containing a csv of our domains and domain requests""" + help = ( "Generates and uploads a domain-metadata.csv file to our S3 bucket " "which is based off of all existing Domains." @@ -65,6 +66,7 @@ class Command(BaseCommand): "report_function": csv_export.DomainRequestExport.export_full_domain_request_report, }, } + # Set the password equal to our content in SECRET_ENCRYPT_METADATA. # For local development, this will be "devpwd" unless otherwise set. override = settings.SECRET_ENCRYPT_METADATA is None and not settings.IS_PRODUCTION @@ -81,7 +83,6 @@ class Command(BaseCommand): attachment_file=encrypted_zip_in_bytes, ) - def get_encrypted_zip(self, zip_filename, reports, password): """Helper function for encrypting the attachment file""" From 4ac9e9d4f3293a45628e3f906345c523f6d92f6b Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:06:31 -0600 Subject: [PATCH 4/9] Remove os --- .../management/commands/email_current_metadata_report.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index b76e77608..0255cc178 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -1,7 +1,6 @@ """Generates current-metadata.csv then uploads to S3 + sends email""" import logging -import os import pyzipper from datetime import datetime From 5acaac7c6c7fe562731c733f50bc57d7b97753a9 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:19:00 -0600 Subject: [PATCH 5/9] Add some docs --- docs/operations/data_migration.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 472362a79..17aa9c606 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -697,3 +697,31 @@ Example: `cf ssh getgov-za` | | Parameter | Description | |:-:|:-------------------------- |:----------------------------------------------------------------------------| | 1 | **debug** | Increases logging detail. Defaults to False. | + +## Email current metadata report + +### Running on sandboxes + +#### Step 1: Login to CloudFoundry +```cf login -a api.fr.cloud.gov --sso``` + +#### Step 2: SSH into your environment +```cf ssh getgov-{space}``` + +Example: `cf ssh getgov-za` + +#### Step 3: Create a shell instance +```/tmp/lifecycle/shell``` + +#### Step 4: Running the script +```./manage.py email_current_metadata_report --emailTo {desired email address}``` + +### Running locally + +#### Step 1: Running the script +```docker-compose exec app ./manage.py email_current_metadata_report --emailTo {desired email address}``` + +##### Parameters +| | Parameter | Description | +|:-:|:-------------------------- |:-----------------------------------------------------------------------------------| +| 1 | **emailTo** | Specifies where the email will be emailed. Defaults to help@get.gov on production. | \ No newline at end of file From 3769113da3343865e7c1ae9693749c53180495bc Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:44:36 -0600 Subject: [PATCH 6/9] Fix bug --- .../management/commands/email_current_metadata_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 0255cc178..595d39215 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -93,8 +93,8 @@ class Command(BaseCommand): f_out.setpassword(str.encode(password)) for report_name, report in reports.items(): logger.info(f"Generating {report_name}") - report = self.write_and_return_report(report["report_function"]) - f_out.writestr(report["report_filename"], report) + report_content = self.write_and_return_report(report["report_function"]) + f_out.writestr(report["report_filename"], report_content) # Get the final report for emailing purposes with open(zip_filename, "rb") as file_data: From aa1958c1432e4659443458294b805efeb9b66be2 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:11:08 -0600 Subject: [PATCH 7/9] Comment out password override for local testing --- .../management/commands/email_current_metadata_report.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 595d39215..82b83ea9b 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -68,8 +68,10 @@ class Command(BaseCommand): # Set the password equal to our content in SECRET_ENCRYPT_METADATA. # For local development, this will be "devpwd" unless otherwise set. - override = settings.SECRET_ENCRYPT_METADATA is None and not settings.IS_PRODUCTION - password = "devpwd" if override else settings.SECRET_ENCRYPT_METADATA + # Uncomment these lines if you want to use this: + # override = settings.SECRET_ENCRYPT_METADATA is None and not settings.IS_PRODUCTION + # password = "devpwd" if override else settings.SECRET_ENCRYPT_METADATA + password = settings.SECRET_ENCRYPT_METADATA encrypted_zip_in_bytes = self.get_encrypted_zip(zip_filename, reports, password) From 5fe084cdabd1b98221192175b29d53f31fef6f30 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:18:37 -0600 Subject: [PATCH 8/9] Add err handling --- .../management/commands/email_current_metadata_report.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 82b83ea9b..773199198 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -72,6 +72,10 @@ class Command(BaseCommand): # override = settings.SECRET_ENCRYPT_METADATA is None and not settings.IS_PRODUCTION # password = "devpwd" if override else settings.SECRET_ENCRYPT_METADATA password = settings.SECRET_ENCRYPT_METADATA + if not password: + raise ValueError( + "No password was specified for this zip file." + ) encrypted_zip_in_bytes = self.get_encrypted_zip(zip_filename, reports, password) From d5cd639b854552f8fba3d5201764c4795aa03a25 Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Mon, 24 Jun 2024 13:21:42 -0600 Subject: [PATCH 9/9] Lint --- .../management/commands/email_current_metadata_report.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index 773199198..905e4a57a 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -73,9 +73,7 @@ class Command(BaseCommand): # password = "devpwd" if override else settings.SECRET_ENCRYPT_METADATA password = settings.SECRET_ENCRYPT_METADATA if not password: - raise ValueError( - "No password was specified for this zip file." - ) + raise ValueError("No password was specified for this zip file.") encrypted_zip_in_bytes = self.get_encrypted_zip(zip_filename, reports, password)