From e78a506d2b2cde785b357c65888bf91595236f4c Mon Sep 17 00:00:00 2001 From: zandercymatics <141044360+zandercymatics@users.noreply.github.com> Date: Wed, 29 Nov 2023 08:36:12 -0700 Subject: [PATCH] Link bucket to generator --- src/api/views.py | 13 +++++++++---- .../commands/generate_current_full_report.py | 16 ++++++++++++---- src/registrar/utility/s3_bucket.py | 13 +++++-------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/api/views.py b/src/api/views.py index 15515dd3a..ddd96f744 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -9,6 +9,8 @@ from login_required import login_not_required from cachetools.func import ttl_cache +from registrar.utility.s3_bucket import S3ClientHelper + DOMAIN_FILE_URL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv" @@ -94,21 +96,24 @@ def available(request, domain=""): @require_http_methods(["GET"]) @login_not_required def get_current_full(request, file_path="migrationdata/current-full.csv"): - return serve_file(file_path) + return serve_file(file_path, "current-full.csv") @require_http_methods(["GET"]) @login_not_required def get_current_federal(request, file_path="migrationdata/current-federal.csv"): - return serve_file(file_path) + return serve_file(file_path, "current-federal.csv") -def serve_file(file_path): +def serve_file(file_path, file_name): """Downloads a file based on a given filepath. Returns a 404 if not found.""" + s3_client = S3ClientHelper() # TODO - #1403, grab from the S3 instance instead + # TODO - check if file exists in s3, not here if os.path.exists(file_path): # Serve the CSV file - response = FileResponse(open(file_path, "rb")) + file = s3_client.get_file(file_name) + response = FileResponse(file) return response else: return HttpResponse("File not found", status=404) diff --git a/src/registrar/management/commands/generate_current_full_report.py b/src/registrar/management/commands/generate_current_full_report.py index 7db334186..f9ed12ba5 100644 --- a/src/registrar/management/commands/generate_current_full_report.py +++ b/src/registrar/management/commands/generate_current_full_report.py @@ -4,6 +4,7 @@ import os from django.core.management import BaseCommand from registrar.utility import csv_export +from registrar.utility.s3_bucket import S3ClientHelper logger = logging.getLogger(__name__) @@ -23,20 +24,27 @@ class Command(BaseCommand): def handle(self, **options): """Grabs the directory then creates current-full.csv in that directory""" + file_name = "current-full.csv" # Ensures a slash is added directory = os.path.join(options.get("directory"), "") check_path = options.get("checkpath") logger.info("Generating report...") - self.generate_current_full_report(directory, check_path) - logger.info(f"Success! Created {directory}current-full.csv") + self.generate_current_full_report(directory, file_name, check_path) - def generate_current_full_report(self, directory, check_path): + file_path = os.path.join(directory, file_name) + logger.info(f"Success! Created {file_path}") + + def generate_current_full_report(self, directory, file_name, check_path): """Creates a current-full.csv file under the specified directory""" + s3_client = S3ClientHelper() # TODO - #1403, push to the S3 instance instead - file_path = os.path.join(directory, "current-full.csv") + file_path = os.path.join(directory, file_name) + # TODO - Don't genererate a useless file with open(file_path, "w") as file: csv_export.export_data_full_to_csv(file) if check_path and not os.path.exists(file_path): raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") + + s3_client.upload_file(file_path, file_name) diff --git a/src/registrar/utility/s3_bucket.py b/src/registrar/utility/s3_bucket.py index af59fdb67..959c6b23a 100644 --- a/src/registrar/utility/s3_bucket.py +++ b/src/registrar/utility/s3_bucket.py @@ -25,15 +25,10 @@ class S3ClientHelper: except Exception as exc: raise S3ClientError("Could not access the S3 client.") from exc - #self.bucket_name = - print("here:") - bucket = self.list_objects() - print(bucket) - def get_bucket_name(self): """Gets the name of our S3 Bucket""" return settings.AWS_S3_BUCKET_NAME - + def list_objects(self): """Returns a list of the top 1000 objects within our S3 instance""" try: @@ -49,14 +44,16 @@ class S3ClientHelper: except Exception as exc: raise S3ClientError("Couldn't upload file") from exc return response - + def get_file(self, file_name, decode_to_utf=False): """Gets a file to our S3 instance and returns the file content""" try: response = self.boto_client.get_object(Bucket=self.get_bucket_name(), Key=file_name) except Exception as exc: raise S3ClientError("Couldn't get file") from exc + file_content = response["Body"].read() if decode_to_utf: return file_content.decode("utf-8") - return file_content + else: + return file_content