mirror of
https://github.com/cisagov/manage.get.gov.git
synced 2025-05-19 10:59:21 +02:00
Link bucket to generator
This commit is contained in:
parent
f5974e00dc
commit
e78a506d2b
3 changed files with 26 additions and 16 deletions
|
@ -9,6 +9,8 @@ from login_required import login_not_required
|
||||||
|
|
||||||
from cachetools.func import ttl_cache
|
from cachetools.func import ttl_cache
|
||||||
|
|
||||||
|
from registrar.utility.s3_bucket import S3ClientHelper
|
||||||
|
|
||||||
|
|
||||||
DOMAIN_FILE_URL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv"
|
DOMAIN_FILE_URL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv"
|
||||||
|
|
||||||
|
@ -94,21 +96,24 @@ def available(request, domain=""):
|
||||||
@require_http_methods(["GET"])
|
@require_http_methods(["GET"])
|
||||||
@login_not_required
|
@login_not_required
|
||||||
def get_current_full(request, file_path="migrationdata/current-full.csv"):
|
def get_current_full(request, file_path="migrationdata/current-full.csv"):
|
||||||
return serve_file(file_path)
|
return serve_file(file_path, "current-full.csv")
|
||||||
|
|
||||||
|
|
||||||
@require_http_methods(["GET"])
|
@require_http_methods(["GET"])
|
||||||
@login_not_required
|
@login_not_required
|
||||||
def get_current_federal(request, file_path="migrationdata/current-federal.csv"):
|
def get_current_federal(request, file_path="migrationdata/current-federal.csv"):
|
||||||
return serve_file(file_path)
|
return serve_file(file_path, "current-federal.csv")
|
||||||
|
|
||||||
|
|
||||||
def serve_file(file_path):
|
def serve_file(file_path, file_name):
|
||||||
"""Downloads a file based on a given filepath. Returns a 404 if not found."""
|
"""Downloads a file based on a given filepath. Returns a 404 if not found."""
|
||||||
|
s3_client = S3ClientHelper()
|
||||||
# TODO - #1403, grab from the S3 instance instead
|
# TODO - #1403, grab from the S3 instance instead
|
||||||
|
# TODO - check if file exists in s3, not here
|
||||||
if os.path.exists(file_path):
|
if os.path.exists(file_path):
|
||||||
# Serve the CSV file
|
# Serve the CSV file
|
||||||
response = FileResponse(open(file_path, "rb"))
|
file = s3_client.get_file(file_name)
|
||||||
|
response = FileResponse(file)
|
||||||
return response
|
return response
|
||||||
else:
|
else:
|
||||||
return HttpResponse("File not found", status=404)
|
return HttpResponse("File not found", status=404)
|
||||||
|
|
|
@ -4,6 +4,7 @@ import os
|
||||||
|
|
||||||
from django.core.management import BaseCommand
|
from django.core.management import BaseCommand
|
||||||
from registrar.utility import csv_export
|
from registrar.utility import csv_export
|
||||||
|
from registrar.utility.s3_bucket import S3ClientHelper
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -23,20 +24,27 @@ class Command(BaseCommand):
|
||||||
|
|
||||||
def handle(self, **options):
|
def handle(self, **options):
|
||||||
"""Grabs the directory then creates current-full.csv in that directory"""
|
"""Grabs the directory then creates current-full.csv in that directory"""
|
||||||
|
file_name = "current-full.csv"
|
||||||
# Ensures a slash is added
|
# Ensures a slash is added
|
||||||
directory = os.path.join(options.get("directory"), "")
|
directory = os.path.join(options.get("directory"), "")
|
||||||
check_path = options.get("checkpath")
|
check_path = options.get("checkpath")
|
||||||
logger.info("Generating report...")
|
logger.info("Generating report...")
|
||||||
|
|
||||||
self.generate_current_full_report(directory, check_path)
|
self.generate_current_full_report(directory, file_name, check_path)
|
||||||
logger.info(f"Success! Created {directory}current-full.csv")
|
|
||||||
|
|
||||||
def generate_current_full_report(self, directory, check_path):
|
file_path = os.path.join(directory, file_name)
|
||||||
|
logger.info(f"Success! Created {file_path}")
|
||||||
|
|
||||||
|
def generate_current_full_report(self, directory, file_name, check_path):
|
||||||
"""Creates a current-full.csv file under the specified directory"""
|
"""Creates a current-full.csv file under the specified directory"""
|
||||||
|
s3_client = S3ClientHelper()
|
||||||
# TODO - #1403, push to the S3 instance instead
|
# TODO - #1403, push to the S3 instance instead
|
||||||
file_path = os.path.join(directory, "current-full.csv")
|
file_path = os.path.join(directory, file_name)
|
||||||
|
# TODO - Don't genererate a useless file
|
||||||
with open(file_path, "w") as file:
|
with open(file_path, "w") as file:
|
||||||
csv_export.export_data_full_to_csv(file)
|
csv_export.export_data_full_to_csv(file)
|
||||||
|
|
||||||
if check_path and not os.path.exists(file_path):
|
if check_path and not os.path.exists(file_path):
|
||||||
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")
|
raise FileNotFoundError(f"Could not find newly created file at '{file_path}'")
|
||||||
|
|
||||||
|
s3_client.upload_file(file_path, file_name)
|
||||||
|
|
|
@ -25,11 +25,6 @@ class S3ClientHelper:
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise S3ClientError("Could not access the S3 client.") from exc
|
raise S3ClientError("Could not access the S3 client.") from exc
|
||||||
|
|
||||||
#self.bucket_name =
|
|
||||||
print("here:")
|
|
||||||
bucket = self.list_objects()
|
|
||||||
print(bucket)
|
|
||||||
|
|
||||||
def get_bucket_name(self):
|
def get_bucket_name(self):
|
||||||
"""Gets the name of our S3 Bucket"""
|
"""Gets the name of our S3 Bucket"""
|
||||||
return settings.AWS_S3_BUCKET_NAME
|
return settings.AWS_S3_BUCKET_NAME
|
||||||
|
@ -56,7 +51,9 @@ class S3ClientHelper:
|
||||||
response = self.boto_client.get_object(Bucket=self.get_bucket_name(), Key=file_name)
|
response = self.boto_client.get_object(Bucket=self.get_bucket_name(), Key=file_name)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise S3ClientError("Couldn't get file") from exc
|
raise S3ClientError("Couldn't get file") from exc
|
||||||
|
|
||||||
file_content = response["Body"].read()
|
file_content = response["Body"].read()
|
||||||
if decode_to_utf:
|
if decode_to_utf:
|
||||||
return file_content.decode("utf-8")
|
return file_content.decode("utf-8")
|
||||||
return file_content
|
else:
|
||||||
|
return file_content
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue