diff --git a/docs/operations/data_migration.md b/docs/operations/data_migration.md index 472362a79..17aa9c606 100644 --- a/docs/operations/data_migration.md +++ b/docs/operations/data_migration.md @@ -697,3 +697,31 @@ Example: `cf ssh getgov-za` | | Parameter | Description | |:-:|:-------------------------- |:----------------------------------------------------------------------------| | 1 | **debug** | Increases logging detail. Defaults to False. | + +## Email current metadata report + +### Running on sandboxes + +#### Step 1: Login to CloudFoundry +```cf login -a api.fr.cloud.gov --sso``` + +#### Step 2: SSH into your environment +```cf ssh getgov-{space}``` + +Example: `cf ssh getgov-za` + +#### Step 3: Create a shell instance +```/tmp/lifecycle/shell``` + +#### Step 4: Running the script +```./manage.py email_current_metadata_report --emailTo {desired email address}``` + +### Running locally + +#### Step 1: Running the script +```docker-compose exec app ./manage.py email_current_metadata_report --emailTo {desired email address}``` + +##### Parameters +| | Parameter | Description | +|:-:|:-------------------------- |:-----------------------------------------------------------------------------------| +| 1 | **emailTo** | Specifies where the email will be emailed. Defaults to help@get.gov on production. | \ No newline at end of file diff --git a/src/registrar/forms/user_profile.py b/src/registrar/forms/user_profile.py index 3dd8cbdce..682e1a5df 100644 --- a/src/registrar/forms/user_profile.py +++ b/src/registrar/forms/user_profile.py @@ -10,6 +10,8 @@ from registrar.models.utility.domain_helper import DomainHelper class UserProfileForm(forms.ModelForm): """Form for updating user profile.""" + redirect = forms.CharField(widget=forms.HiddenInput(), required=False) + class Meta: model = Contact fields = ["first_name", "middle_name", "last_name", "title", "email", "phone"] diff --git a/src/registrar/management/commands/email_current_metadata_report.py b/src/registrar/management/commands/email_current_metadata_report.py index dcaf47b06..905e4a57a 100644 --- a/src/registrar/management/commands/email_current_metadata_report.py +++ b/src/registrar/management/commands/email_current_metadata_report.py @@ -1,7 +1,6 @@ """Generates current-metadata.csv then uploads to S3 + sends email""" import logging -import os import pyzipper from datetime import datetime @@ -9,7 +8,7 @@ from datetime import datetime from django.core.management import BaseCommand from django.conf import settings from registrar.utility import csv_export -from registrar.utility.s3_bucket import S3ClientHelper +from io import StringIO from ...utility.email import send_templated_email @@ -17,89 +16,101 @@ logger = logging.getLogger(__name__) class Command(BaseCommand): + """Emails a encrypted zip file containing a csv of our domains and domain requests""" + help = ( "Generates and uploads a domain-metadata.csv file to our S3 bucket " "which is based off of all existing Domains." ) + current_date = datetime.now().strftime("%m%d%Y") def add_arguments(self, parser): """Add our two filename arguments.""" - parser.add_argument("--directory", default="migrationdata", help="Desired directory") parser.add_argument( - "--checkpath", - default=True, - help="Flag that determines if we do a check for os.path.exists. Used for test cases", + "--emailTo", + default=settings.DEFAULT_FROM_EMAIL, + help="Defines where we should email this report", ) def handle(self, **options): """Grabs the directory then creates domain-metadata.csv in that directory""" - file_name = "domain-metadata.csv" - # Ensures a slash is added - directory = os.path.join(options.get("directory"), "") - check_path = options.get("checkpath") + zip_filename = f"domain-metadata-{self.current_date}.zip" + email_to = options.get("emailTo") + + # Don't email to DEFAULT_FROM_EMAIL when not prod. + if not settings.IS_PRODUCTION and email_to == settings.DEFAULT_FROM_EMAIL: + raise ValueError( + "The --emailTo arg must be specified in non-prod environments, " + "and the arg must not equal the DEFAULT_FROM_EMAIL value (aka: help@get.gov)." + ) logger.info("Generating report...") try: - self.email_current_metadata_report(directory, file_name, check_path) + self.email_current_metadata_report(zip_filename, email_to) except Exception as err: # TODO - #1317: Notify operations when auto report generation fails raise err else: - logger.info(f"Success! Created {file_name} and successfully sent out an email!") + logger.info(f"Success! Created {zip_filename} and successfully sent out an email!") - def email_current_metadata_report(self, directory, file_name, check_path): - """Creates a current-metadata.csv file under the specified directory, - then uploads it to a AWS S3 bucket. This is done for resiliency - reasons in the event our application goes down and/or the email - cannot send -- we'll still be able to grab info from the S3 - instance""" - s3_client = S3ClientHelper() - file_path = os.path.join(directory, file_name) + def email_current_metadata_report(self, zip_filename, email_to): + """Emails a password protected zip containing domain-metadata and domain-request-metadata""" + reports = { + "Domain report": { + "report_filename": f"domain-metadata-{self.current_date}.csv", + "report_function": csv_export.export_data_type_to_csv, + }, + "Domain request report": { + "report_filename": f"domain-request-metadata-{self.current_date}.csv", + "report_function": csv_export.DomainRequestExport.export_full_domain_request_report, + }, + } - # Generate a file locally for upload - with open(file_path, "w") as file: - csv_export.export_data_type_to_csv(file) + # Set the password equal to our content in SECRET_ENCRYPT_METADATA. + # For local development, this will be "devpwd" unless otherwise set. + # Uncomment these lines if you want to use this: + # override = settings.SECRET_ENCRYPT_METADATA is None and not settings.IS_PRODUCTION + # password = "devpwd" if override else settings.SECRET_ENCRYPT_METADATA + password = settings.SECRET_ENCRYPT_METADATA + if not password: + raise ValueError("No password was specified for this zip file.") - if check_path and not os.path.exists(file_path): - raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") - - s3_client.upload_file(file_path, file_name) - - # Set zip file name - current_date = datetime.now().strftime("%m%d%Y") - current_filename = f"domain-metadata-{current_date}.zip" - - # Pre-set zip file name - encrypted_metadata_output = current_filename - - # Set context for the subject - current_date_str = datetime.now().strftime("%Y-%m-%d") - - # Encrypt the metadata - encrypted_metadata_in_bytes = self._encrypt_metadata( - s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA) - ) + encrypted_zip_in_bytes = self.get_encrypted_zip(zip_filename, reports, password) # Send the metadata file that is zipped send_templated_email( template_name="emails/metadata_body.txt", subject_template_name="emails/metadata_subject.txt", - to_address=settings.DEFAULT_FROM_EMAIL, - context={"current_date_str": current_date_str}, - attachment_file=encrypted_metadata_in_bytes, + to_address=email_to, + context={"current_date_str": datetime.now().strftime("%Y-%m-%d")}, + attachment_file=encrypted_zip_in_bytes, ) - def _encrypt_metadata(self, input_file, output_file, password): + def get_encrypted_zip(self, zip_filename, reports, password): """Helper function for encrypting the attachment file""" - current_date = datetime.now().strftime("%m%d%Y") - current_filename = f"domain-metadata-{current_date}.csv" + # Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities and faster # We could also use compression=pyzipper.ZIP_LZMA if we are looking for smaller file size with pyzipper.AESZipFile( - output_file, "w", compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES + zip_filename, "w", compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES ) as f_out: - f_out.setpassword(password) - f_out.writestr(current_filename, input_file) - with open(output_file, "rb") as file_data: + f_out.setpassword(str.encode(password)) + for report_name, report in reports.items(): + logger.info(f"Generating {report_name}") + report_content = self.write_and_return_report(report["report_function"]) + f_out.writestr(report["report_filename"], report_content) + + # Get the final report for emailing purposes + with open(zip_filename, "rb") as file_data: attachment_in_bytes = file_data.read() + return attachment_in_bytes + + def write_and_return_report(self, report_function): + """Writes a report to a StringIO object given a report_function and returns the string.""" + report_bytes = StringIO() + report_function(report_bytes) + + # Rewind the buffer to the beginning after writing + report_bytes.seek(0) + return report_bytes.read() diff --git a/src/registrar/models/domain_request.py b/src/registrar/models/domain_request.py index 6327bcd77..9265a726e 100644 --- a/src/registrar/models/domain_request.py +++ b/src/registrar/models/domain_request.py @@ -17,6 +17,8 @@ from .utility.time_stamped_model import TimeStampedModel from ..utility.email import send_templated_email, EmailSendingError from itertools import chain +from waffle.decorators import flag_is_active + logger = logging.getLogger(__name__) @@ -1165,19 +1167,21 @@ class DomainRequest(TimeStampedModel): def _is_policy_acknowledgement_complete(self): return self.is_policy_acknowledged is not None - def _is_general_form_complete(self): + def _is_general_form_complete(self, request): + has_profile_feature_flag = flag_is_active(request, "profile_feature") return ( self._is_organization_name_and_address_complete() and self._is_senior_official_complete() and self._is_requested_domain_complete() and self._is_purpose_complete() - and self._is_submitter_complete() + # NOTE: This flag leaves submitter as empty (request wont submit) hence preset to True + and (self._is_submitter_complete() if not has_profile_feature_flag else True) and self._is_other_contacts_complete() and self._is_additional_details_complete() and self._is_policy_acknowledgement_complete() ) - def _form_complete(self): + def _form_complete(self, request): match self.generic_org_type: case DomainRequest.OrganizationChoices.FEDERAL: is_complete = self._is_federal_complete() @@ -1198,8 +1202,6 @@ class DomainRequest(TimeStampedModel): case _: # NOTE: Shouldn't happen, this is only if somehow they didn't choose an org type is_complete = False - - if not is_complete or not self._is_general_form_complete(): + if not is_complete or not self._is_general_form_complete(request): return False - return True diff --git a/src/registrar/registrar_middleware.py b/src/registrar/registrar_middleware.py index f5274c99f..82ee9d0fc 100644 --- a/src/registrar/registrar_middleware.py +++ b/src/registrar/registrar_middleware.py @@ -45,6 +45,7 @@ class CheckUserProfileMiddleware: self.setup_page = reverse("finish-user-profile-setup") self.profile_page = reverse("user-profile") self.logout_page = reverse("logout") + self.regular_excluded_pages = [ self.setup_page, self.logout_page, @@ -56,6 +57,14 @@ class CheckUserProfileMiddleware: "/admin", ] + self.excluded_pages = { + self.setup_page: self.regular_excluded_pages, + self.profile_page: self.other_excluded_pages, + } + + def _get_excluded_pages(self, page): + return self.excluded_pages.get(page, []) + def __call__(self, request): response = self.get_response(request) return response @@ -72,16 +81,16 @@ class CheckUserProfileMiddleware: return None if request.user.is_authenticated: + profile_page = self.profile_page + if request.user.verification_type == User.VerificationTypeChoices.REGULAR: + profile_page = self.setup_page if hasattr(request.user, "finished_setup") and not request.user.finished_setup: - if request.user.verification_type == User.VerificationTypeChoices.REGULAR: - return self._handle_regular_user_setup_not_finished(request) - else: - return self._handle_other_user_setup_not_finished(request) + return self._handle_user_setup_not_finished(request, profile_page) # Continue processing the view return None - def _handle_regular_user_setup_not_finished(self, request): + def _handle_user_setup_not_finished(self, request, profile_page): """Redirects the given user to the finish setup page. We set the "redirect" query param equal to where the user wants to go. @@ -97,7 +106,7 @@ class CheckUserProfileMiddleware: custom_redirect = "domain-request:" if request.path == "/request/" else None # Don't redirect on excluded pages (such as the setup page itself) - if not any(request.path.startswith(page) for page in self.regular_excluded_pages): + if not any(request.path.startswith(page) for page in self._get_excluded_pages(profile_page)): # Preserve the original query parameters, and coerce them into a dict query_params = parse_qs(request.META["QUERY_STRING"]) @@ -107,23 +116,13 @@ class CheckUserProfileMiddleware: query_params["redirect"] = custom_redirect # Add our new query param, while preserving old ones - new_setup_page = replace_url_queryparams(self.setup_page, query_params) if query_params else self.setup_page + new_setup_page = replace_url_queryparams(profile_page, query_params) if query_params else profile_page return HttpResponseRedirect(new_setup_page) else: # Process the view as normal return None - def _handle_other_user_setup_not_finished(self, request): - """Redirects the given user to the profile page to finish setup.""" - - # Don't redirect on excluded pages (such as the setup page itself) - if not any(request.path.startswith(page) for page in self.other_excluded_pages): - return HttpResponseRedirect(self.profile_page) - else: - # Process the view as normal - return None - class CheckPortfolioMiddleware: """ diff --git a/src/registrar/templates/domain_request_intro.html b/src/registrar/templates/domain_request_intro.html index 2bfeeeef1..370ea2b2b 100644 --- a/src/registrar/templates/domain_request_intro.html +++ b/src/registrar/templates/domain_request_intro.html @@ -18,7 +18,7 @@ completing your domain request might take around 15 minutes.
{% if has_profile_feature_flag %}While reviewing your domain request, we may need to reach out with questions. We’ll also email you when we complete our review If the contact information below is not correct, visit your profile to make updates.
+While reviewing your domain request, we may need to reach out with questions. We’ll also email you when we complete our review If the contact information below is not correct, visit your profile to make updates.
{% include "includes/profile_information.html" with user=user%} {% endif %} diff --git a/src/registrar/templates/finish_profile_setup.html b/src/registrar/templates/finish_profile_setup.html index f8070551b..6e35ad5da 100644 --- a/src/registrar/templates/finish_profile_setup.html +++ b/src/registrar/templates/finish_profile_setup.html @@ -5,7 +5,7 @@ {# Disable the redirect #} {% block logo %} - {% include "includes/gov_extended_logo.html" with logo_clickable=confirm_changes %} + {% include "includes/gov_extended_logo.html" with logo_clickable=user_finished_setup %} {% endblock %} {# Add the new form #} @@ -16,5 +16,5 @@ {% endblock content_bottom %} {% block footer %} - {% include "includes/footer.html" with show_manage_your_domains=confirm_changes %} + {% include "includes/footer.html" with show_manage_your_domains=user_finished_setup %} {% endblock footer %} diff --git a/src/registrar/templates/includes/finish_profile_form.html b/src/registrar/templates/includes/finish_profile_form.html index a40534b48..d43ff812c 100644 --- a/src/registrar/templates/includes/finish_profile_form.html +++ b/src/registrar/templates/includes/finish_profile_form.html @@ -33,6 +33,8 @@ Your contact information + + {% with show_edit_button=True show_readonly=True group_classes="usa-form-editable usa-form-editable--no-border padding-top-2" %} {% input_with_errors form.full_name %} {% endwith %} @@ -78,7 +80,7 @@ - {% if confirm_changes and going_to_specific_page %} + {% if user_finished_setup and going_to_specific_page %} diff --git a/src/registrar/templates/includes/profile_form.html b/src/registrar/templates/includes/profile_form.html index cb3e734bf..966b92b01 100644 --- a/src/registrar/templates/includes/profile_form.html +++ b/src/registrar/templates/includes/profile_form.html @@ -19,6 +19,9 @@