zydronium/manage.get.gov
1
0
Fork 0
mirror of https://github.com/cisagov/manage.get.gov.git synced 2025-06-08 13:34:42 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1856 from cisagov/rh/1727-metadata-emaill

Browse source 
[on getgov-backup] ISSUE #1727: You've Got Mail -- "Full Domain Metadata" Encrypted + Emailed to Team Inbox
This commit is contained in:
Rebecca H 2024-03-13 16:49:49 -07:00 • committed by GitHub
commit fce679f417
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 272 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/Pipfile
View file

@ -29,6 +29,7 @@ django-login-required-middleware = "*"
greenlet = "*"
gevent = "*"
fred-epplib = {git = "https://github.com/cisagov/epplib.git", ref = "master"}
pyzipper="*"
tblib = "*"
[dev-packages]
@ -44,4 +45,4 @@ django-webtest = "*"
types-cachetools = "*"
boto3-mocking = "*"
boto3-stubs = "*"
django-model2puml = "*"
django-model2puml = "*"

65
src/Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "b5d93b1b9ccafc37019276a222957544bab3f1f46b5dab8a0f2ffc2e5c9e1678"
"sha256": "082a951f15bb26a28f2dca7e0840fdf61518b3d90c42d77a310f982344cbd1dc"
},
"pipfile-spec": 6,
"requires": {},
@ -32,20 +32,20 @@
},
"boto3": {
"hashes": [
"sha256:8b3f5cc7fbedcbb22271c328039df8a6ab343001e746e0cdb24774c426cadcf8",
"sha256:f201b6a416f809283d554c652211eecec9fe3a52ed4063dab3f3e7aea7571d9c"
"sha256:300888f0c1b6f32f27f85a9aa876f50f46514ec619647af7e4d20db74d339714",
"sha256:b26928f9a21cf3649cea20a59061340f3294c6e7785ceb6e1a953eb8010dc3ba"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==1.34.54"
"version": "==1.34.56"
},
"botocore": {
"hashes": [
"sha256:4061ff4be3efcf53547ebadf2c94d419dfc8be7beec24e9fa1819599ffd936fa",
"sha256:bf215d93e9d5544c593962780d194e74c6ee40b883d0b885e62ef35fc0ec01e5"
"sha256:bffeb71ab21d47d4ecf947d9bdb2fbd1b0bbd0c27742cea7cf0b77b701c41d9f",
"sha256:fff66e22a5589c2d58fba57d1d95c334ce771895e831f80365f6cff6453285ec"
],
"markers": "python_version >= '3.8'",
"version": "==1.34.54"
"version": "==1.34.56"
},
"cachetools": {
"hashes": [
@ -376,20 +376,20 @@
"django"
],
"hashes": [
"sha256:cc421ddb143fa30183568164755aa113a160e555cd19e97e664c478662032c24",
"sha256:feeaf28f17fd0499f9cd7c0fcf408c6d82c308e69e335eb92d09322fc9ed8138"
"sha256:069727a8f73d8ba8d033d3cd95c0da231d44f38f1da773bf076cef168d312ee8",
"sha256:e0bcfd41c718c07a7db422f9109e490746450da38793fe4ee197f397b9343435"
],
"markers": "python_version >= '3.8'",
"version": "==10.3.0"
"version": "==11.0.0"
},
"faker": {
"hashes": [
"sha256:117ce1a2805c1bc5ca753b3dc6f9d567732893b2294b827d3164261ee8f20267",
"sha256:458d93580de34403a8dec1e8d5e6be2fee96c4deca63b95d71df7a6a80a690de"
"sha256:2456d674f40bd51eb3acbf85221277027822e529a90cc826453d9a25dff932b1",
"sha256:ea6f784c40730de0f77067e49e78cdd590efb00bec3d33f577492262206c17fc"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==23.3.0"
"version": "==24.0.0"
},
"fred-epplib": {
"git": "https://github.com/cisagov/epplib.git",
@ -708,11 +708,11 @@
},
"marshmallow": {
"hashes": [
"sha256:20f53be28c6e374a711a16165fb22a8dc6003e3f7cda1285e3ca777b9193885b",
"sha256:e7997f83571c7fd476042c2c188e4ee8a78900ca5e74bd9c8097afa56624e9bd"
"sha256:4e65e9e0d80fc9e609574b9983cf32579f305c718afb30d7233ab818571768c3",
"sha256:f085493f79efb0644f270a9bf2892843142d80d7174bbbd2f3713f2a589dc633"
],
"markers": "python_version >= '3.8'",
"version": "==3.21.0"
"version": "==3.21.1"
},
"oic": {
"hashes": [
@ -994,6 +994,15 @@
"markers": "python_version >= '3.8'",
"version": "==1.0.1"
},
"pyzipper": {
"hashes": [
"sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc",
"sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87"
],
"index": "pypi",
"markers": "python_version >= '3.4'",
"version": "==0.3.6"
},
"requests": {
"hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
@ -1186,12 +1195,12 @@
},
"boto3": {
"hashes": [
"sha256:8b3f5cc7fbedcbb22271c328039df8a6ab343001e746e0cdb24774c426cadcf8",
"sha256:f201b6a416f809283d554c652211eecec9fe3a52ed4063dab3f3e7aea7571d9c"
"sha256:300888f0c1b6f32f27f85a9aa876f50f46514ec619647af7e4d20db74d339714",
"sha256:b26928f9a21cf3649cea20a59061340f3294c6e7785ceb6e1a953eb8010dc3ba"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==1.34.54"
"version": "==1.34.56"
},
"boto3-mocking": {
"hashes": [
@ -1204,28 +1213,28 @@
},
"boto3-stubs": {
"hashes": [
"sha256:7db5194e47f76e0010cd00b6ad9725db114d6a3fd04e52ceed3ef1181fe326bc",
"sha256:c7b2e8b99f4896cf1226df47d4badaaa8df7426008c96a428bf00205695669e9"
"sha256:627f8eca69d832581ee1676d39df099a2a2e3a86d6b3ebd21c81c5f11ed6a6fa",
"sha256:a87e7ecbab6235ec371b4363027e57483bca349a9cd5c891f40db81dadfa273e"
],
"index": "pypi",
"markers": "python_version >= '3.8'",
"version": "==1.34.54"
"version": "==1.34.56"
},
"botocore": {
"hashes": [
"sha256:4061ff4be3efcf53547ebadf2c94d419dfc8be7beec24e9fa1819599ffd936fa",
"sha256:bf215d93e9d5544c593962780d194e74c6ee40b883d0b885e62ef35fc0ec01e5"
"sha256:bffeb71ab21d47d4ecf947d9bdb2fbd1b0bbd0c27742cea7cf0b77b701c41d9f",
"sha256:fff66e22a5589c2d58fba57d1d95c334ce771895e831f80365f6cff6453285ec"
],
"markers": "python_version >= '3.8'",
"version": "==1.34.54"
"version": "==1.34.56"
},
"botocore-stubs": {
"hashes": [
"sha256:958f0084322dc9e549f73151b686fa51b15858fb2b3a573b9f4367f073fff463",
"sha256:bcc35bfbd14d1261813681c40108f2ce85fdf082c15b0a04016d3c22dd93b73f"
"sha256:018e001e3add5eb1828ef444b45fb8c9faf695e08334031bf2d96853cd9af703",
"sha256:25468ba6983987b704b1856bb155f297f576e6d4a690b021ab0c7122889ba907"
],
"markers": "python_version >= '3.8' and python_version < '4.0'",
"version": "==1.34.54"
"version": "==1.34.56"
},
"click": {
"hashes": [

2
src/docker-compose.yml
View file

@ -58,6 +58,8 @@ services:
- AWS_S3_SECRET_ACCESS_KEY
- AWS_S3_REGION
- AWS_S3_BUCKET_NAME
# File encryption credentials
- SECRET_ENCRYPT_METADATA
stdin_open: true
tty: true
ports:

4
src/registrar/config/settings.py
View file

@ -74,6 +74,9 @@ secret_aws_s3_key_id = secret("access_key_id", None) or secret("AWS_S3_ACCESS_KE
secret_aws_s3_key = secret("secret_access_key", None) or secret("AWS_S3_SECRET_ACCESS_KEY", None)
secret_aws_s3_bucket_name = secret("bucket", None) or secret("AWS_S3_BUCKET_NAME", None)
# Passphrase for the encrypted metadata email
secret_encrypt_metadata = secret("SECRET_ENCRYPT_METADATA", None)
secret_registry_cl_id = secret("REGISTRY_CL_ID")
secret_registry_password = secret("REGISTRY_PASSWORD")
secret_registry_cert = b64decode(secret("REGISTRY_CERT", ""))
@ -94,6 +97,7 @@ DEBUG = env_debug
# Controls production specific feature toggles
IS_PRODUCTION = env_is_production
SECRET_ENCRYPT_METADATA = secret_encrypt_metadata
# Applications are modular pieces of code.
# They are provided by Django, by third-parties, or by yourself.

105
src/registrar/management/commands/email_current_metadata_report.py Normal file
View file

@ -0,0 +1,105 @@
"""Generates current-metadata.csv then uploads to S3 + sends email"""
import logging
import os
import pyzipper
from datetime import datetime
from django.core.management import BaseCommand
from django.conf import settings
from registrar.utility import csv_export
from registrar.utility.s3_bucket import S3ClientHelper
from ...utility.email import send_templated_email
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = (
"Generates and uploads a domain-metadata.csv file to our S3 bucket "
"which is based off of all existing Domains."
)
def add_arguments(self, parser):
"""Add our two filename arguments."""
parser.add_argument("--directory", default="migrationdata", help="Desired directory")
parser.add_argument(
"--checkpath",
default=True,
help="Flag that determines if we do a check for os.path.exists. Used for test cases",
)
def handle(self, **options):
"""Grabs the directory then creates domain-metadata.csv in that directory"""
file_name = "domain-metadata.csv"
# Ensures a slash is added
directory = os.path.join(options.get("directory"), "")
check_path = options.get("checkpath")
logger.info("Generating report...")
try:
self.email_current_metadata_report(directory, file_name, check_path)
except Exception as err:
# TODO - #1317: Notify operations when auto report generation fails
raise err
else:
logger.info(f"Success! Created {file_name} and successfully sent out an email!")
def email_current_metadata_report(self, directory, file_name, check_path):
"""Creates a current-metadata.csv file under the specified directory,
then uploads it to a AWS S3 bucket. This is done for resiliency
reasons in the event our application goes down and/or the email
cannot send -- we'll still be able to grab info from the S3
instance"""
s3_client = S3ClientHelper()
file_path = os.path.join(directory, file_name)
# Generate a file locally for upload
with open(file_path, "w") as file:
csv_export.export_data_type_to_csv(file)
if check_path and not os.path.exists(file_path):
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")
s3_client.upload_file(file_path, file_name)
# Set zip file name
current_date = datetime.now().strftime("%m%d%Y")
current_filename = f"domain-metadata-{current_date}.zip"
# Pre-set zip file name
encrypted_metadata_output = current_filename
# Set context for the subject
current_date_str = datetime.now().strftime("%Y-%m-%d")
# Encrypt the metadata
encrypted_metadata_in_bytes = self._encrypt_metadata(
s3_client.get_file(file_name), encrypted_metadata_output, str.encode(settings.SECRET_ENCRYPT_METADATA)
)
# Send the metadata file that is zipped
send_templated_email(
template_name="emails/metadata_body.txt",
subject_template_name="emails/metadata_subject.txt",
to_address=settings.DEFAULT_FROM_EMAIL,
context={"current_date_str": current_date_str},
attachment_file=encrypted_metadata_in_bytes,
)
def _encrypt_metadata(self, input_file, output_file, password):
"""Helper function for encrypting the attachment file"""
current_date = datetime.now().strftime("%m%d%Y")
current_filename = f"domain-metadata-{current_date}.csv"
# Using ZIP_DEFLATED bc it's a more common compression method supported by most zip utilities and faster
# We could also use compression=pyzipper.ZIP_LZMA if we are looking for smaller file size
with pyzipper.AESZipFile(
output_file, "w", compression=pyzipper.ZIP_DEFLATED, encryption=pyzipper.WZ_AES
) as f_out:
f_out.setpassword(password)
f_out.writestr(current_filename, input_file)
with open(output_file, "rb") as file_data:
attachment_in_bytes = file_data.read()
return attachment_in_bytes

1
src/registrar/templates/emails/metadata_body.txt Normal file
View file

@ -0,0 +1 @@
An export of all .gov metadata.

2
src/registrar/templates/emails/metadata_subject.txt Normal file
View file

@ -0,0 +1,2 @@
Domain metadata - {{current_date_str}}

32
src/registrar/tests/test_emails.py
View file

@ -5,7 +5,8 @@ from unittest.mock import MagicMock
from django.test import TestCase
from .common import completed_domain_request, less_console_noise
from datetime import datetime
from registrar.utility import email
import boto3_mocking # type: ignore
@ -182,3 +183,32 @@ class TestEmails(TestCase):
self.assertNotIn("Anything else", body)
# spacing should be right between adjacent elements
self.assertRegex(body, r"5557\n\n----")
@boto3_mocking.patching
def test_send_email_with_attachment(self):
with boto3_mocking.clients.handler_for("ses", self.mock_client_class):
sender_email = "sender@example.com"
recipient_email = "recipient@example.com"
subject = "Test Subject"
body = "Test Body"
attachment_file = b"Attachment file content"
current_date = datetime.now().strftime("%m%d%Y")
current_filename = f"domain-metadata-{current_date}.zip"
email.send_email_with_attachment(
sender_email, recipient_email, subject, body, attachment_file, self.mock_client
)
# Assert that the `send_raw_email` method of the mocked SES client was called with the expected params
self.mock_client.send_raw_email.assert_called_once()
# Get the args passed to the `send_raw_email` method
call_args = self.mock_client.send_raw_email.call_args[1]
# Assert that the attachment filename is correct
self.assertEqual(call_args["RawMessage"]["Data"].count(f'filename="{current_filename}"'), 1)
# Assert that the attachment content is encrypted
self.assertIn("Content-Type: application/octet-stream", call_args["RawMessage"]["Data"])
self.assertIn("Content-Transfer-Encoding: base64", call_args["RawMessage"]["Data"])
self.assertIn("Content-Disposition: attachment;", call_args["RawMessage"]["Data"])
self.assertNotIn("Attachment file content", call_args["RawMessage"]["Data"])

66
src/registrar/utility/email.py
View file

@ -2,8 +2,12 @@
import boto3
import logging
from datetime import datetime
from django.conf import settings
from django.template.loader import get_template
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
logger = logging.getLogger(__name__)
@ -15,7 +19,14 @@ class EmailSendingError(RuntimeError):
pass
def send_templated_email(template_name: str, subject_template_name: str, to_address: str, bcc_address="", context={}):
def send_templated_email(
template_name: str,
subject_template_name: str,
to_address: str,
bcc_address="",
context={},
attachment_file: str = None,
):
"""Send an email built from a template to one email address.
template_name and subject_template_name are relative to the same template
@ -45,15 +56,50 @@ def send_templated_email(template_name: str, subject_template_name: str, to_addr
destination["BccAddresses"] = [bcc_address]
try:
ses_client.send_email(
FromEmailAddress=settings.DEFAULT_FROM_EMAIL,
Destination=destination,
Content={
"Simple": {
"Subject": {"Data": subject},
"Body": {"Text": {"Data": email_body}},
if attachment_file is None:
ses_client.send_email(
FromEmailAddress=settings.DEFAULT_FROM_EMAIL,
Destination=destination,
Content={
"Simple": {
"Subject": {"Data": subject},
"Body": {"Text": {"Data": email_body}},
},
},
},
)
)
else:
ses_client = boto3.client(
"ses",
region_name=settings.AWS_REGION,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
config=settings.BOTO_CONFIG,
)
send_email_with_attachment(
settings.DEFAULT_FROM_EMAIL, to_address, subject, email_body, attachment_file, ses_client
)
except Exception as exc:
raise EmailSendingError("Could not send SES email.") from exc
def send_email_with_attachment(sender, recipient, subject, body, attachment_file, ses_client):
# Create a multipart/mixed parent container
msg = MIMEMultipart("mixed")
msg["Subject"] = subject
msg["From"] = sender
msg["To"] = recipient
# Add the text part
text_part = MIMEText(body, "plain")
msg.attach(text_part)
# Add the attachment part
attachment_part = MIMEApplication(attachment_file)
# Adding attachment header + filename that the attachment will be called
current_date = datetime.now().strftime("%m%d%Y")
current_filename = f"domain-metadata-{current_date}.zip"
attachment_part.add_header("Content-Disposition", f'attachment; filename="{current_filename}"')
msg.attach(attachment_part)
response = ses_client.send_raw_email(Source=sender, Destinations=[recipient], RawMessage={"Data": msg.as_string()})
return response

11
src/requirements.txt
View file

@ -1,8 +1,8 @@
-i https://pypi.python.org/simple
annotated-types==0.6.0; python_version >= '3.8'
asgiref==3.7.2; python_version >= '3.7'
boto3==1.34.54; python_version >= '3.8'
botocore==1.34.54; python_version >= '3.8'
boto3==1.34.56; python_version >= '3.8'
botocore==1.34.56; python_version >= '3.8'
cachetools==5.3.3; python_version >= '3.7'
certifi==2024.2.2; python_version >= '3.6'
cfenv==0.5.3
@ -22,8 +22,8 @@ django-fsm==2.8.1
django-login-required-middleware==0.9.0
django-phonenumber-field[phonenumberslite]==7.3.0; python_version >= '3.8'
django-widget-tweaks==1.5.0; python_version >= '3.8'
environs[django]==10.3.0; python_version >= '3.8'
faker==23.3.0; python_version >= '3.8'
environs[django]==11.0.0; python_version >= '3.8'
faker==24.0.0; python_version >= '3.8'
fred-epplib@ git+https://github.com/cisagov/epplib.git@d56d183f1664f34c40ca9716a3a9a345f0ef561c
furl==2.1.3
future==1.0.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
@ -35,7 +35,7 @@ jmespath==1.0.1; python_version >= '3.7'
lxml==5.1.0; python_version >= '3.6'
mako==1.3.2; python_version >= '3.8'
markupsafe==2.1.5; python_version >= '3.7'
marshmallow==3.21.0; python_version >= '3.8'
marshmallow==3.21.1; python_version >= '3.8'
oic==1.6.1; python_version ~= '3.7'
orderedmultidict==1.0.1
packaging==23.2; python_version >= '3.7'
@ -49,6 +49,7 @@ pydantic-settings==2.2.1; python_version >= '3.8'
pyjwkest==1.4.2
python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
python-dotenv==1.0.1; python_version >= '3.8'
pyzipper==0.3.6; python_version >= '3.4'
requests==2.31.0; python_version >= '3.7'
s3transfer==0.10.0; python_version >= '3.8'
setuptools==69.1.1; python_version >= '3.8'