Link bucket to generator

This commit is contained in:
zandercymatics 2023-11-29 08:36:12 -07:00
parent f5974e00dc
commit e78a506d2b
No known key found for this signature in database
GPG key ID: FF4636ABEC9682B7
3 changed files with 26 additions and 16 deletions

View file

@ -9,6 +9,8 @@ from login_required import login_not_required
from cachetools.func import ttl_cache from cachetools.func import ttl_cache
from registrar.utility.s3_bucket import S3ClientHelper
DOMAIN_FILE_URL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv" DOMAIN_FILE_URL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv"
@ -94,21 +96,24 @@ def available(request, domain=""):
@require_http_methods(["GET"]) @require_http_methods(["GET"])
@login_not_required @login_not_required
def get_current_full(request, file_path="migrationdata/current-full.csv"): def get_current_full(request, file_path="migrationdata/current-full.csv"):
return serve_file(file_path) return serve_file(file_path, "current-full.csv")
@require_http_methods(["GET"]) @require_http_methods(["GET"])
@login_not_required @login_not_required
def get_current_federal(request, file_path="migrationdata/current-federal.csv"): def get_current_federal(request, file_path="migrationdata/current-federal.csv"):
return serve_file(file_path) return serve_file(file_path, "current-federal.csv")
def serve_file(file_path): def serve_file(file_path, file_name):
"""Downloads a file based on a given filepath. Returns a 404 if not found.""" """Downloads a file based on a given filepath. Returns a 404 if not found."""
s3_client = S3ClientHelper()
# TODO - #1403, grab from the S3 instance instead # TODO - #1403, grab from the S3 instance instead
# TODO - check if file exists in s3, not here
if os.path.exists(file_path): if os.path.exists(file_path):
# Serve the CSV file # Serve the CSV file
response = FileResponse(open(file_path, "rb")) file = s3_client.get_file(file_name)
response = FileResponse(file)
return response return response
else: else:
return HttpResponse("File not found", status=404) return HttpResponse("File not found", status=404)

View file

@ -4,6 +4,7 @@ import os
from django.core.management import BaseCommand from django.core.management import BaseCommand
from registrar.utility import csv_export from registrar.utility import csv_export
from registrar.utility.s3_bucket import S3ClientHelper
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -23,20 +24,27 @@ class Command(BaseCommand):
def handle(self, **options): def handle(self, **options):
"""Grabs the directory then creates current-full.csv in that directory""" """Grabs the directory then creates current-full.csv in that directory"""
file_name = "current-full.csv"
# Ensures a slash is added # Ensures a slash is added
directory = os.path.join(options.get("directory"), "") directory = os.path.join(options.get("directory"), "")
check_path = options.get("checkpath") check_path = options.get("checkpath")
logger.info("Generating report...") logger.info("Generating report...")
self.generate_current_full_report(directory, check_path) self.generate_current_full_report(directory, file_name, check_path)
logger.info(f"Success! Created {directory}current-full.csv")
def generate_current_full_report(self, directory, check_path): file_path = os.path.join(directory, file_name)
logger.info(f"Success! Created {file_path}")
def generate_current_full_report(self, directory, file_name, check_path):
"""Creates a current-full.csv file under the specified directory""" """Creates a current-full.csv file under the specified directory"""
s3_client = S3ClientHelper()
# TODO - #1403, push to the S3 instance instead # TODO - #1403, push to the S3 instance instead
file_path = os.path.join(directory, "current-full.csv") file_path = os.path.join(directory, file_name)
# TODO - Don't genererate a useless file
with open(file_path, "w") as file: with open(file_path, "w") as file:
csv_export.export_data_full_to_csv(file) csv_export.export_data_full_to_csv(file)
if check_path and not os.path.exists(file_path): if check_path and not os.path.exists(file_path):
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'") raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")
s3_client.upload_file(file_path, file_name)

View file

@ -25,11 +25,6 @@ class S3ClientHelper:
except Exception as exc: except Exception as exc:
raise S3ClientError("Could not access the S3 client.") from exc raise S3ClientError("Could not access the S3 client.") from exc
#self.bucket_name =
print("here:")
bucket = self.list_objects()
print(bucket)
def get_bucket_name(self): def get_bucket_name(self):
"""Gets the name of our S3 Bucket""" """Gets the name of our S3 Bucket"""
return settings.AWS_S3_BUCKET_NAME return settings.AWS_S3_BUCKET_NAME
@ -56,7 +51,9 @@ class S3ClientHelper:
response = self.boto_client.get_object(Bucket=self.get_bucket_name(), Key=file_name) response = self.boto_client.get_object(Bucket=self.get_bucket_name(), Key=file_name)
except Exception as exc: except Exception as exc:
raise S3ClientError("Couldn't get file") from exc raise S3ClientError("Couldn't get file") from exc
file_content = response["Body"].read() file_content = response["Body"].read()
if decode_to_utf: if decode_to_utf:
return file_content.decode("utf-8") return file_content.decode("utf-8")
else:
return file_content return file_content