mirror of
https://github.com/cisagov/manage.get.gov.git
synced 2025-05-21 20:09:23 +02:00
Update yaml to include metadata, readme with new directions an
d pseudocode for s3 and SES work
This commit is contained in:
parent
2817a3ed5b
commit
fb1735e23f
4 changed files with 85 additions and 4 deletions
9
.github/workflows/daily-csv-upload.yaml
vendored
9
.github/workflows/daily-csv-upload.yaml
vendored
|
@ -31,3 +31,12 @@ jobs:
|
||||||
cf_space: ${{ secrets.CF_REPORT_ENV }}
|
cf_space: ${{ secrets.CF_REPORT_ENV }}
|
||||||
cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_full_report' --name full"
|
cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_full_report' --name full"
|
||||||
|
|
||||||
|
- name: Generate current-metadata.csv
|
||||||
|
uses: cloud-gov/cg-cli-tools@main
|
||||||
|
with:
|
||||||
|
cf_username: ${{ secrets[env.CF_USERNAME] }}
|
||||||
|
cf_password: ${{ secrets[env.CF_PASSWORD] }}
|
||||||
|
cf_org: cisa-dotgov
|
||||||
|
cf_space: ${{ secrets.CF_REPORT_ENV }}
|
||||||
|
cf_command: "run-task getgov-${{ secrets.CF_REPORT_ENV }} --command 'python manage.py generate_current_metadata_report' --name metadata"
|
||||||
|
|
||||||
|
|
|
@ -330,11 +330,12 @@ To associate a S3 instance to your sandbox, follow these steps:
|
||||||
3. Click `Services` on the application nav bar
|
3. Click `Services` on the application nav bar
|
||||||
4. Add a new service (plus symbol)
|
4. Add a new service (plus symbol)
|
||||||
5. Click `Marketplace Service`
|
5. Click `Marketplace Service`
|
||||||
6. On the `Select the service` dropdown, select `s3`
|
6. For Space, put in your sandbox initials
|
||||||
7. Under the dropdown on `Select Plan`, select `basic-sandbox`
|
7. On the `Select the service` dropdown, select `s3`
|
||||||
8. Under `Service Instance` enter `getgov-s3` for the name
|
8. Under the dropdown on `Select Plan`, select `basic-sandbox`
|
||||||
|
9. Under `Service Instance` enter `getgov-s3` for the name and leave the other fields empty
|
||||||
|
|
||||||
See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI.
|
See this [resource](https://cloud.gov/docs/services/s3/) for information on associating an S3 instance with your sandbox through the CLI. The basic commands should be `cf bind-service getgov-<your-initials> <getgov-s3>` and `cf restage getgov-<your-initials>`.
|
||||||
|
|
||||||
### Testing your S3 instance locally
|
### Testing your S3 instance locally
|
||||||
To test the S3 bucket associated with your sandbox, you will need to add four additional variables to your `.env` file. These are as follows:
|
To test the S3 bucket associated with your sandbox, you will need to add four additional variables to your `.env` file. These are as follows:
|
||||||
|
|
|
@ -0,0 +1,67 @@
|
||||||
|
"""Generates current-metadata.csv then uploads to S3 + sends email"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
|
||||||
|
from django.core.management import BaseCommand
|
||||||
|
from registrar.utility import csv_export
|
||||||
|
from registrar.utility.s3_bucket import S3ClientHelper
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = (
|
||||||
|
"Generates and uploads a current-metadata.csv file to our S3 bucket " "which is based off of all existing Domains."
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_arguments(self, parser):
|
||||||
|
"""Add our two filename arguments."""
|
||||||
|
parser.add_argument("--directory", default="migrationdata", help="Desired directory")
|
||||||
|
parser.add_argument(
|
||||||
|
"--checkpath",
|
||||||
|
default=True,
|
||||||
|
help="Flag that determines if we do a check for os.path.exists. Used for test cases",
|
||||||
|
)
|
||||||
|
|
||||||
|
def handle(self, **options):
|
||||||
|
"""Grabs the directory then creates current-metadata.csv in that directory"""
|
||||||
|
file_name = "current-metadata.csv"
|
||||||
|
# Ensures a slash is added
|
||||||
|
directory = os.path.join(options.get("directory"), "")
|
||||||
|
check_path = options.get("checkpath")
|
||||||
|
|
||||||
|
logger.info("Generating report...")
|
||||||
|
try:
|
||||||
|
self.generate_current_metadata_report(directory, file_name, check_path)
|
||||||
|
except Exception as err:
|
||||||
|
# TODO - #1317: Notify operations when auto report generation fails
|
||||||
|
raise err
|
||||||
|
else:
|
||||||
|
logger.info(f"Success! Created {file_name}")
|
||||||
|
|
||||||
|
def generate_current_metadata_report(self, directory, file_name, check_path):
|
||||||
|
"""Creates a current-full.csv file under the specified directory,
|
||||||
|
then uploads it to a AWS S3 bucket"""
|
||||||
|
s3_client = S3ClientHelper()
|
||||||
|
file_path = os.path.join(directory, file_name)
|
||||||
|
|
||||||
|
# Generate a file locally for upload
|
||||||
|
with open(file_path, "w") as file:
|
||||||
|
csv_export.export_data_type_to_csv(file)
|
||||||
|
|
||||||
|
if check_path and not os.path.exists(file_path):
|
||||||
|
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")
|
||||||
|
|
||||||
|
# Upload this generated file for our S3 instance
|
||||||
|
s3_client.upload_file(file_path, file_name)
|
||||||
|
"""
|
||||||
|
We want to make sure to upload to s3 for back up
|
||||||
|
And now we also want to get the file and encrypt it so we can send it in an email
|
||||||
|
"""
|
||||||
|
# metadata_file = s3_client.get_file(file_name)
|
||||||
|
# metadata_file.encryptthisthingherewithpyzipper
|
||||||
|
# email.blasend_email(metadata_file)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -41,6 +41,7 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr
|
||||||
raise EmailSendingError("Could not access the SES client.") from exc
|
raise EmailSendingError("Could not access the SES client.") from exc
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
#if not attachment:
|
||||||
ses_client.send_email(
|
ses_client.send_email(
|
||||||
FromEmailAddress=settings.DEFAULT_FROM_EMAIL,
|
FromEmailAddress=settings.DEFAULT_FROM_EMAIL,
|
||||||
Destination={"ToAddresses": [to_address]},
|
Destination={"ToAddresses": [to_address]},
|
||||||
|
@ -51,5 +52,8 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
# else: # has attachment
|
||||||
|
# same as above but figure out how to attach a file
|
||||||
|
# via boto3 "boto3 SES file attachment"
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise EmailSendingError("Could not send SES email.") from exc
|
raise EmailSendingError("Could not send SES email.") from exc
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue