Link bucket to generator

This commit is contained in:
zandercymatics 2023-11-29 08:36:12 -07:00
parent f5974e00dc
commit e78a506d2b
No known key found for this signature in database
GPG key ID: FF4636ABEC9682B7
3 changed files with 26 additions and 16 deletions

View file

@ -9,6 +9,8 @@ from login_required import login_not_required
from cachetools.func import ttl_cache
from registrar.utility.s3_bucket import S3ClientHelper
DOMAIN_FILE_URL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv"
@ -94,21 +96,24 @@ def available(request, domain=""):
@require_http_methods(["GET"])
@login_not_required
def get_current_full(request, file_path="migrationdata/current-full.csv"):
return serve_file(file_path)
return serve_file(file_path, "current-full.csv")
@require_http_methods(["GET"])
@login_not_required
def get_current_federal(request, file_path="migrationdata/current-federal.csv"):
return serve_file(file_path)
return serve_file(file_path, "current-federal.csv")
def serve_file(file_path):
def serve_file(file_path, file_name):
"""Downloads a file based on a given filepath. Returns a 404 if not found."""
s3_client = S3ClientHelper()
# TODO - #1403, grab from the S3 instance instead
# TODO - check if file exists in s3, not here
if os.path.exists(file_path):
# Serve the CSV file
response = FileResponse(open(file_path, "rb"))
file = s3_client.get_file(file_name)
response = FileResponse(file)
return response
else:
return HttpResponse("File not found", status=404)

View file

@ -4,6 +4,7 @@ import os
from django.core.management import BaseCommand
from registrar.utility import csv_export
from registrar.utility.s3_bucket import S3ClientHelper
logger = logging.getLogger(__name__)
@ -23,20 +24,27 @@ class Command(BaseCommand):
def handle(self, **options):
"""Grabs the directory then creates current-full.csv in that directory"""
file_name = "current-full.csv"
# Ensures a slash is added
directory = os.path.join(options.get("directory"), "")
check_path = options.get("checkpath")
logger.info("Generating report...")
self.generate_current_full_report(directory, check_path)
logger.info(f"Success! Created {directory}current-full.csv")
self.generate_current_full_report(directory, file_name, check_path)
def generate_current_full_report(self, directory, check_path):
file_path = os.path.join(directory, file_name)
logger.info(f"Success! Created {file_path}")
def generate_current_full_report(self, directory, file_name, check_path):
"""Creates a current-full.csv file under the specified directory"""
s3_client = S3ClientHelper()
# TODO - #1403, push to the S3 instance instead
file_path = os.path.join(directory, "current-full.csv")
file_path = os.path.join(directory, file_name)
# TODO - Don't genererate a useless file
with open(file_path, "w") as file:
csv_export.export_data_full_to_csv(file)
if check_path and not os.path.exists(file_path):
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")
s3_client.upload_file(file_path, file_name)

View file

@ -25,15 +25,10 @@ class S3ClientHelper:
except Exception as exc:
raise S3ClientError("Could not access the S3 client.") from exc
#self.bucket_name =
print("here:")
bucket = self.list_objects()
print(bucket)
def get_bucket_name(self):
"""Gets the name of our S3 Bucket"""
return settings.AWS_S3_BUCKET_NAME
def list_objects(self):
"""Returns a list of the top 1000 objects within our S3 instance"""
try:
@ -49,14 +44,16 @@ class S3ClientHelper:
except Exception as exc:
raise S3ClientError("Couldn't upload file") from exc
return response
def get_file(self, file_name, decode_to_utf=False):
"""Gets a file to our S3 instance and returns the file content"""
try:
response = self.boto_client.get_object(Bucket=self.get_bucket_name(), Key=file_name)
except Exception as exc:
raise S3ClientError("Couldn't get file") from exc
file_content = response["Body"].read()
if decode_to_utf:
return file_content.decode("utf-8")
return file_content
else:
return file_content