Update yaml to include metadata, readme with new directions an

d pseudocode for s3 and SES work
This commit is contained in:
Rebecca Hsieh 2024-02-22 16:51:36 -08:00
parent 2817a3ed5b
commit fb1735e23f
No known key found for this signature in database
4 changed files with 85 additions and 4 deletions

View file

@ -31,3 +31,12 @@ jobs:
cf_space: ${{ secrets.CF_REPORT_ENV }} cf_space: ${{ secrets.CF_REPORT_ENV }}
cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_full_report' --name full" cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_full_report' --name full"
- name: Generate current-metadata.csv
uses: cloud-gov/cg-cli-tools@main
with:
cf_username: ${{ secrets[env.CF_USERNAME] }}
cf_password: ${{ secrets[env.CF_PASSWORD] }}
cf_org: cisa-dotgov
cf_space: ${{ secrets.CF_REPORT_ENV }}
cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_metadata_report' --name metadata"

View file

@ -330,11 +330,12 @@ To associate a S3 instance to your sandbox, follow these steps:
3. Click `Services` on the application nav bar 3. Click `Services` on the application nav bar
4. Add a new service (plus symbol) 4. Add a new service (plus symbol)
5. Click `Marketplace Service` 5. Click `Marketplace Service`
6. On the `Select the service` dropdown, select `s3` 6. For Space, put in your sandbox initials
7. Under the dropdown on `Select Plan`, select `basic-sandbox` 7. On the `Select the service` dropdown, select `s3`
8. Under `Service Instance` enter `getgov-s3` for the name 8. Under the dropdown on `Select Plan`, select `basic-sandbox`
9. Under `Service Instance` enter `getgov-s3` for the name and leave the other fields empty
See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI. See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI. The basic commands should be `cf bind-service getgov-<your-initials> <getgov-s3>` and `cf restage getgov-<your-initials>`.
### Testing your S3 instance locally ### Testing your S3 instance locally
To test the S3 bucket associated with your sandbox, you will need to add four additional variables to your `.env` file. These are as follows: To test the S3 bucket associated with your sandbox, you will need to add four additional variables to your `.env` file. These are as follows:

View file

@ -0,0 +1,67 @@
"""Generates current-metadata.csv then uploads to S3 + sends email"""
import logging
import os
from django.core.management import BaseCommand
from registrar.utility import csv_export
from registrar.utility.s3_bucket import S3ClientHelper
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = (
"Generates and uploads a current-metadata.csv file to our S3 bucket " "which is based off of all existing Domains."
)
def add_arguments(self, parser):
"""Add our two filename arguments."""
parser.add_argument("--directory", default="migrationdata", help="Desired directory")
parser.add_argument(
"--checkpath",
default=True,
help="Flag that determines if we do a check for os.path.exists. Used for test cases",
)
def handle(self, **options):
"""Grabs the directory then creates current-metadata.csv in that directory"""
file_name = "current-metadata.csv"
# Ensures a slash is added
directory = os.path.join(options.get("directory"), "")
check_path = options.get("checkpath")
logger.info("Generating report...")
try:
self.generate_current_metadata_report(directory, file_name, check_path)
except Exception as err:
# TODO - #1317: Notify operations when auto report generation fails
raise err
else:
logger.info(f"Success! Created {file_name}")
def generate_current_metadata_report(self, directory, file_name, check_path):
"""Creates a current-full.csv file under the specified directory,
then uploads it to a AWS S3 bucket"""
s3_client = S3ClientHelper()
file_path = os.path.join(directory, file_name)
# Generate a file locally for upload
with open(file_path, "w") as file:
csv_export.export_data_type_to_csv(file)
if check_path and not os.path.exists(file_path):
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")
# Upload this generated file for our S3 instance
s3_client.upload_file(file_path, file_name)
"""
We want to make sure to upload to s3 for back up
And now we also want to get the file and encrypt it so we can send it in an email
"""
# metadata_file = s3_client.get_file(file_name)
# metadata_file.encryptthisthingherewithpyzipper
# email.blasend_email(metadata_file)

View file

@ -41,6 +41,7 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr
raise EmailSendingError("Could not access the SES client.") from exc raise EmailSendingError("Could not access the SES client.") from exc
try: try:
#if not attachment:
ses_client.send_email( ses_client.send_email(
FromEmailAddress=settings.DEFAULT_FROM_EMAIL, FromEmailAddress=settings.DEFAULT_FROM_EMAIL,
Destination={"ToAddresses": [to_address]}, Destination={"ToAddresses": [to_address]},
@ -51,5 +52,8 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr
}, },
}, },
) )
# else: # has attachment
# same as above but figure out how to attach a file
# via boto3 "boto3 SES file attachment"
except Exception as exc: except Exception as exc:
raise EmailSendingError("Could not send SES email.") from exc raise EmailSendingError("Could not send SES email.") from exc