diff --git a/.github/workflows/daily-csv-upload.yaml b/.github/workflows/daily-csv-upload.yaml new file mode 100644 index 000000000..729b5bd16 --- /dev/null +++ b/.github/workflows/daily-csv-upload.yaml @@ -0,0 +1,49 @@ +name: Upload current-full.csv and current-federal.csv + +on: + push: + paths-ignore: + - 'docs/**' + - '**.md' + - '.gitignore' + branches: + - rjm + pull_request: + paths-ignore: + - 'docs/**' + - '**.md' + - '.gitignore' + branches: + - rjm + schedule: + # Runs every day at 5 AM UTC + - cron: '0 5 * * *' + +jobs: + upload_reports: + runs-on: ubuntu-latest + if: github.event_name == 'schedule' + steps: + - uses: actions/checkout@v3 + + - name: Install CF CLI + run: | + curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&source=github" | tar -zx + sudo mv cf /usr/local/bin + + - name: Login to cloud.gov + run: | + cf api https://api.fr.cloud.gov + cf auth ${{ secrets.CF_USERNAME }} ${{ secrets.CF_PASSWORD }} + cf target -o ${{ secrets.CF_ORG }} -s ${{ secrets.CF_SPACE }} + + - name: Run task + run: cf run-task my-app "/tmp/lifecycle/shell -c './manage.py generate_current_full_and_federal_reports.py'" + + - name: Commit and push CSV files + run: | + git config --global user.name 'GitHub Actions' + git config --global user.email 'actions@github.com' + git add current-full.csv current-federal.csv + git commit -m "Update CSV files" + git push \ No newline at end of file diff --git a/src/api/report_views.py b/src/api/report_views.py deleted file mode 100644 index 2e479a8a1..000000000 --- a/src/api/report_views.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Internal API views""" -from django.apps import apps -from django.views.decorators.http import require_http_methods -from django.http import FileResponse, JsonResponse - -import requests - - -from registrar.utility import csv_export -from login_required import login_not_required - -@require_http_methods(["GET"]) -@login_not_required -def get_current_full(request): - # Generate the CSV file - with open("current-full.csv", "w") as file: - csv_export.export_data_full_to_csv(file) - - # Serve the CSV file - response = FileResponse(open('current-full.csv', 'rb')) - return response - -@require_http_methods(["GET"]) -@login_not_required -def get_current_federal(request): - # Generate the CSV file - with open("current-federal.csv", "w") as file: - csv_export.export_data_federal_to_csv(file) - - # Serve the CSV file - response = FileResponse(open('current-federal.csv', 'rb')) - return response diff --git a/src/api/views.py b/src/api/views.py index 2cb23a9b2..1ed8a0888 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -1,7 +1,7 @@ """Internal API views""" from django.apps import apps from django.views.decorators.http import require_http_methods -from django.http import JsonResponse +from django.http import FileResponse, HttpResponse, JsonResponse import requests @@ -89,3 +89,26 @@ def available(request, domain=""): return JsonResponse({"available": False, "message": DOMAIN_API_MESSAGES["unavailable"]}) except Exception: return JsonResponse({"available": False, "message": DOMAIN_API_MESSAGES["error"]}) + +@require_http_methods(["GET"]) +@login_not_required +def get_current_full(request): + # Open the CSV file + file_path = './migrationData/current-full.csv' + return serve_file(file_path) + +@require_http_methods(["GET"]) +@login_not_required +def get_current_federal(request): + # Open the CSV file + file_path = './migrationData/current-federal.csv' + return serve_file(file_path) + +def serve_file(file_path): + """Downloads a file based on a given filepath. Returns a 404 if not found.""" + if os.path.exists(file_path): + # Serve the CSV file + response = FileResponse(open(file_path, 'rb')) + return response + else: + return HttpResponse("File not found", status=404) \ No newline at end of file diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index e71f1388e..6ded44913 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -11,8 +11,8 @@ from django.views.generic import RedirectView from registrar import views from registrar.views.application import Step from registrar.views.utility import always_404 -from api.views import available -from api.report_views import get_current_federal, get_current_full +from api.views import available, get_current_federal, get_current_full + APPLICATION_NAMESPACE = views.ApplicationWizard.URL_NAMESPACE application_urls = [ diff --git a/src/registrar/management/commands/generate_current_federal_report.py b/src/registrar/management/commands/generate_current_federal_report.py new file mode 100644 index 000000000..f07a35c65 --- /dev/null +++ b/src/registrar/management/commands/generate_current_federal_report.py @@ -0,0 +1,35 @@ +"""Generates current-full.csv and current-federal.csv then uploads them to the desired URL.""" +import glob +import logging + +import os +import shutil + +from django.core.management import BaseCommand + +from registrar.management.commands.utility.terminal_helper import TerminalHelper +from registrar.utility import csv_export + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "" + + def add_arguments(self, parser): + """Add our two filename arguments.""" + parser.add_argument("--directory", default="migrationdata", help="Desired directory") + + def handle(self, **options): + # Ensures a slash is added + directory = os.path.join(options.get("directory"), "") + logger.info("Generating report...") + self.generate_current_federal_report(directory) + logger.info(f"Success! Created {directory}current-federal.csv") + + def generate_current_federal_report(self, directory): + """Creates a current-full.csv file under the migrationdata/ directory""" + file_path = os.path.join(directory, "current-federal.csv") + with open(file_path, "w") as file: + csv_export.export_data_federal_to_csv(file) diff --git a/src/registrar/management/commands/generate_current_full_report.py b/src/registrar/management/commands/generate_current_full_report.py new file mode 100644 index 000000000..49f2127a8 --- /dev/null +++ b/src/registrar/management/commands/generate_current_full_report.py @@ -0,0 +1,35 @@ +"""Generates current-full.csv and current-federal.csv then uploads them to the desired URL.""" +import glob +import logging + +import os +import shutil + +from django.core.management import BaseCommand + +from registrar.management.commands.utility.terminal_helper import TerminalHelper +from registrar.utility import csv_export + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "" + + def add_arguments(self, parser): + """Add our two filename arguments.""" + parser.add_argument("--directory", default="migrationdata", help="Desired directory") + + def handle(self, **options): + # Ensures a slash is added + directory = os.path.join(options.get("directory"), "") + logger.info("Generating report...") + self.generate_current_full_report(directory) + logger.info(f"Success! Created {directory}current-full.csv") + + def generate_current_full_report(self, directory): + """Creates a current-full.csv file under the migrationdata/ directory""" + file_path = os.path.join(directory, "current-full.csv") + with open(file_path, "w") as file: + csv_export.export_data_full_to_csv(file)