diff --git a/.github/SECURITY.md b/.github/SECURITY.md index fc27feff3..e6fba722b 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,5 +1,5 @@ -* If you've found a security or privacy issue on the **.gov top-level domain infrastructure**, submit it to our [vulnerabilty disclosure form](https://forms.office.com/Pages/ResponsePage.aspx?id=bOfNPG2UEkq7evydCEI1SqHke9Gh6wJEl3kQ5EjWUKlUMTZZS1lBVkxHUzZURFpLTkE2NEJFVlhVRi4u) or email dotgov@cisa.dhs.gov. -* If you see a security or privacy issue on **an individual .gov domain**, check [current-full.csv](https://flatgithub.com/cisagov/dotgov-data/blob/main/?filename=current-full.csv) or [Whois](https://domains.dotgov.gov/dotgov-web/registration/whois.xhtml) (same data) to check whether the domain has a security contact to report your finding directly. You are welcome to Cc dotgov@cisa.dhs.gov on the email. - * If you are unable to find a contact or receive no response from the security contact, email dotgov@cisa.dhs.gov. +* If you've found a security or privacy issue on the **.gov top-level domain infrastructure**, submit it to our [vulnerabilty disclosure form](https://forms.office.com/Pages/ResponsePage.aspx?id=bOfNPG2UEkq7evydCEI1SqHke9Gh6wJEl3kQ5EjWUKlUMTZZS1lBVkxHUzZURFpLTkE2NEJFVlhVRi4u) or email help@get.gov. +* If you see a security or privacy issue on **an individual .gov domain**, check [current-full.csv](https://flatgithub.com/cisagov/dotgov-data/blob/main/?filename=current-full.csv) to see whether the domain has a security contact to report your finding directly. You are welcome to Cc help@get.gov on the email. + * If you are unable to find a contact or receive no response from the security contact, email help@get.gov. Note that most federal (executive branch) agencies maintain a [vulnerability disclosure policy](https://github.com/cisagov/vdp-in-fceb/). diff --git a/src/registrar/admin.py b/src/registrar/admin.py index aed06021c..364ae81f6 100644 --- a/src/registrar/admin.py +++ b/src/registrar/admin.py @@ -983,6 +983,10 @@ class DomainAdmin(ListHeaderAdmin): "name", "organization_type", "state", + "expiration_date", + "created_at", + "first_ready", + "deleted", ] # this ordering effects the ordering of results @@ -1001,7 +1005,7 @@ class DomainAdmin(ListHeaderAdmin): search_help_text = "Search by domain name." change_form_template = "django/admin/domain_change_form.html" change_list_template = "django/admin/domain_change_list.html" - readonly_fields = ["state", "expiration_date"] + readonly_fields = ["state", "expiration_date", "first_ready", "deleted"] # Table ordering ordering = ["name"] @@ -1246,8 +1250,9 @@ admin.site.register(models.DomainInvitation, DomainInvitationAdmin) admin.site.register(models.DomainInformation, DomainInformationAdmin) admin.site.register(models.Domain, DomainAdmin) admin.site.register(models.DraftDomain, DraftDomainAdmin) -admin.site.register(models.Host, MyHostAdmin) -admin.site.register(models.Nameserver, MyHostAdmin) +# Host and HostIP removed from django admin because changes in admin +# do not propogate to registry and logic not applied +# admin.site.register(models.Host, MyHostAdmin) admin.site.register(models.Website, WebsiteAdmin) admin.site.register(models.PublicContact, AuditedAdmin) admin.site.register(models.DomainApplication, DomainApplicationAdmin) diff --git a/src/registrar/assets/js/get-gov-admin.js b/src/registrar/assets/js/get-gov-admin.js index 53eeb22a3..cdbbc83ee 100644 --- a/src/registrar/assets/js/get-gov-admin.js +++ b/src/registrar/assets/js/get-gov-admin.js @@ -275,3 +275,39 @@ function enableRelatedWidgetButtons(changeLink, deleteLink, viewLink, elementPk, viewLink.setAttribute('href', viewLink.getAttribute('data-href-template').replace('__fk__', elementPk)); viewLink.setAttribute('title', viewLink.getAttribute('title-template').replace('selected item', elementText)); } + +/** An IIFE for admin in DjangoAdmin to listen to clicks on the growth report export button, + * attach the seleted start and end dates to a url that'll trigger the view, and finally + * redirect to that url. +*/ +(function (){ + + // Get the current date in the format YYYY-MM-DD + var currentDate = new Date().toISOString().split('T')[0]; + + // Default the value of the start date input field to the current date + let startDateInput =document.getElementById('start'); + startDateInput.value = currentDate; + + // Default the value of the end date input field to the current date + let endDateInput =document.getElementById('end'); + endDateInput.value = currentDate; + + let exportGrowthReportButton = document.getElementById('exportLink'); + + if (exportGrowthReportButton) { + exportGrowthReportButton.addEventListener('click', function() { + // Get the selected start and end dates + let startDate = startDateInput.value; + let endDate = endDateInput.value; + let exportUrl = document.getElementById('exportLink').dataset.exportUrl; + + // Build the URL with parameters + exportUrl += "?start_date=" + startDate + "&end_date=" + endDate; + + // Redirect to the export URL + window.location.href = exportUrl; + }); + } + +})(); diff --git a/src/registrar/assets/sass/_theme/_admin.scss b/src/registrar/assets/sass/_theme/_admin.scss index 3afc81a35..a3d631243 100644 --- a/src/registrar/assets/sass/_theme/_admin.scss +++ b/src/registrar/assets/sass/_theme/_admin.scss @@ -136,10 +136,17 @@ html[data-theme="dark"] { } #branding h1, -h1, h2, h3 { +h1, h2, h3, +.module h2 { font-weight: font-weight('bold'); } +.module h3 { + padding: 0; + color: var(--primary); + margin: units(2) 0 units(1) 0; +} + .change-list { .usa-table--striped tbody tr:nth-child(odd) td, .usa-table--striped tbody tr:nth-child(odd) th, diff --git a/src/registrar/config/settings.py b/src/registrar/config/settings.py index c99daf72b..bc46c60ba 100644 --- a/src/registrar/config/settings.py +++ b/src/registrar/config/settings.py @@ -210,7 +210,6 @@ STATICFILES_DIRS = [ TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [BASE_DIR / "registrar" / "templates"], # look for templates inside installed apps # required by django-debug-toolbar "APP_DIRS": True, diff --git a/src/registrar/config/urls.py b/src/registrar/config/urls.py index d30c85ce9..607bf5f61 100644 --- a/src/registrar/config/urls.py +++ b/src/registrar/config/urls.py @@ -9,6 +9,10 @@ from django.urls import include, path from django.views.generic import RedirectView from registrar import views + +from registrar.views.admin_views import ExportData + + from registrar.views.application import Step from registrar.views.utility import always_404 from api.views import available, get_current_federal, get_current_full @@ -49,6 +53,7 @@ urlpatterns = [ "admin/logout/", RedirectView.as_view(pattern_name="logout", permanent=False), ), + path("export_data/", ExportData.as_view(), name="admin_export_data"), path("admin/", admin.site.urls), path( "application//edit/", diff --git a/src/registrar/forms/application_wizard.py b/src/registrar/forms/application_wizard.py index e0bc5fe52..fcf6bda7a 100644 --- a/src/registrar/forms/application_wizard.py +++ b/src/registrar/forms/application_wizard.py @@ -577,12 +577,44 @@ class OtherContactsForm(RegistrarForm): error_messages={"required": "Enter a phone number for this contact."}, ) + def clean(self): + """ + This method overrides the default behavior for forms. + This cleans the form after field validation has already taken place. + In this override, allow for a form which is empty to be considered + valid even though certain required fields have not passed field + validation + """ + + # Set form_is_empty to True initially + form_is_empty = True + for name, field in self.fields.items(): + # get the value of the field from the widget + value = field.widget.value_from_datadict(self.data, self.files, self.add_prefix(name)) + # if any field in the submitted form is not empty, set form_is_empty to False + if value is not None and value != "": + form_is_empty = False + + if form_is_empty: + # clear any errors raised by the form fields + # (before this clean() method is run, each field + # performs its own clean, which could result in + # errors that we wish to ignore at this point) + # + # NOTE: we cannot just clear() the errors list. + # That causes problems. + for field in self.fields: + if field in self.errors: + del self.errors[field] + + return self.cleaned_data + class BaseOtherContactsFormSet(RegistrarFormSet): JOIN = "other_contacts" def should_delete(self, cleaned): - empty = (isinstance(v, str) and not v.strip() for v in cleaned.values()) + empty = (isinstance(v, str) and (v.strip() == "" or v is None) for v in cleaned.values()) return all(empty) def to_database(self, obj: DomainApplication): diff --git a/src/registrar/migrations/0059_delete_nameserver.py b/src/registrar/migrations/0059_delete_nameserver.py new file mode 100644 index 000000000..404f8abf0 --- /dev/null +++ b/src/registrar/migrations/0059_delete_nameserver.py @@ -0,0 +1,15 @@ +# Generated by Django 4.2.7 on 2023-12-21 11:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0058_alter_domaininformation_options"), + ] + + operations = [ + migrations.DeleteModel( + name="Nameserver", + ), + ] diff --git a/src/registrar/migrations/0060_domain_deleted_domain_first_ready.py b/src/registrar/migrations/0060_domain_deleted_domain_first_ready.py new file mode 100644 index 000000000..e4caa1525 --- /dev/null +++ b/src/registrar/migrations/0060_domain_deleted_domain_first_ready.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.7 on 2023-12-29 22:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("registrar", "0059_delete_nameserver"), + ] + + operations = [ + migrations.AddField( + model_name="domain", + name="deleted", + field=models.DateField(editable=False, help_text="Deleted at date", null=True), + ), + migrations.AddField( + model_name="domain", + name="first_ready", + field=models.DateField( + editable=False, help_text="The last time this domain moved into the READY state", null=True + ), + ), + ] diff --git a/src/registrar/models/__init__.py b/src/registrar/models/__init__.py index 1203c7878..6afad5a5c 100644 --- a/src/registrar/models/__init__.py +++ b/src/registrar/models/__init__.py @@ -7,7 +7,6 @@ from .draft_domain import DraftDomain from .host_ip import HostIP from .host import Host from .domain_invitation import DomainInvitation -from .nameserver import Nameserver from .user_domain_role import UserDomainRole from .public_contact import PublicContact from .user import User @@ -24,7 +23,6 @@ __all__ = [ "DomainInvitation", "HostIP", "Host", - "Nameserver", "UserDomainRole", "PublicContact", "User", @@ -41,7 +39,6 @@ auditlog.register(DomainInvitation) auditlog.register(DomainInformation) auditlog.register(HostIP) auditlog.register(Host) -auditlog.register(Nameserver) auditlog.register(UserDomainRole) auditlog.register(PublicContact) auditlog.register(User, m2m_fields=["user_permissions", "groups"]) diff --git a/src/registrar/models/domain.py b/src/registrar/models/domain.py index 7d2086499..1ae9b8029 100644 --- a/src/registrar/models/domain.py +++ b/src/registrar/models/domain.py @@ -10,7 +10,8 @@ from django_fsm import FSMField, transition, TransitionNotAllowed # type: ignor from django.db import models from django.utils import timezone from typing import Any - +from registrar.models.host import Host +from registrar.models.host_ip import HostIP from registrar.utility.errors import ( ActionNotAllowed, @@ -295,13 +296,15 @@ class Domain(TimeStampedModel, DomainHelper): while non-subordinate hosts MUST NOT. """ try: + # attempt to retrieve hosts from registry and store in cache and db hosts = self._get_property("hosts") - except Exception as err: - # Do not raise error when missing nameservers - # this is a standard occurence when a domain - # is first created - logger.info("Domain is missing nameservers %s" % err) - return [] + except Exception: + # If exception raised returning hosts from registry, get from db + hosts = [] + for hostobj in self.host.all(): + host_name = hostobj.name + ips = [ip.address for ip in hostobj.ip.all()] + hosts.append({"name": host_name, "addrs": ips}) # TODO-687 fix this return value hostList = [] @@ -730,8 +733,10 @@ class Domain(TimeStampedModel, DomainHelper): email=contact.email, voice=contact.voice, fax=contact.fax, + auth_info=epp.ContactAuthInfo(pw="2fooBAR123fooBaz"), ) # type: ignore + updateContact.disclose = self._disclose_fields(contact=contact) # type: ignore try: registry.send(updateContact, cleaned=True) except RegistryError as e: @@ -969,6 +974,18 @@ class Domain(TimeStampedModel, DomainHelper): help_text=("Duplication of registry's expiration date saved for ease of reporting"), ) + deleted = DateField( + null=True, + editable=False, + help_text="Deleted at date", + ) + + first_ready = DateField( + null=True, + editable=False, + help_text="The last time this domain moved into the READY state", + ) + def isActive(self): return self.state == Domain.State.CREATED @@ -1298,6 +1315,7 @@ class Domain(TimeStampedModel, DomainHelper): try: logger.info("deletedInEpp()-> inside _delete_domain") self._delete_domain() + self.deleted = timezone.now() except RegistryError as err: logger.error(f"Could not delete domain. Registry returned error: {err}") raise err @@ -1341,6 +1359,11 @@ class Domain(TimeStampedModel, DomainHelper): """ logger.info("Changing to ready state") logger.info("able to transition to ready state") + # if self.first_ready is not None, this means that this + # domain was READY, then not READY, then is READY again. + # We do not want to overwrite first_ready. + if self.first_ready is None: + self.first_ready = timezone.now() @transition( field="state", @@ -1605,6 +1628,8 @@ class Domain(TimeStampedModel, DomainHelper): cache = self._extract_data_from_response(data_response) cleaned = self._clean_cache(cache, data_response) self._update_hosts_and_contacts(cleaned, fetch_hosts, fetch_contacts) + if fetch_hosts: + self._update_hosts_and_ips_in_db(cleaned) self._update_dates(cleaned) self._cache = cleaned @@ -1651,7 +1676,11 @@ class Domain(TimeStampedModel, DomainHelper): return dnssec_data def _update_hosts_and_contacts(self, cleaned, fetch_hosts, fetch_contacts): - """Capture and store old hosts and contacts from cache if they don't exist""" + """ + Update hosts and contacts if fetch_hosts and/or fetch_contacts. + Additionally, capture and cache old hosts and contacts from cache if they + don't exist in cleaned + """ old_cache_hosts = self._cache.get("hosts") old_cache_contacts = self._cache.get("contacts") @@ -1666,6 +1695,50 @@ class Domain(TimeStampedModel, DomainHelper): if old_cache_contacts is not None: cleaned["contacts"] = old_cache_contacts + def _update_hosts_and_ips_in_db(self, cleaned): + """Update hosts and host_ips in database if retrieved from registry. + Only called when fetch_hosts is True. + + Parameters: + self: the domain to be updated with hosts and ips from cleaned + cleaned: dict containing hosts. Hosts are provided as a list of dicts, e.g. + [{"name": "ns1.example.com",}, {"name": "ns1.example.gov"}, "addrs": ["0.0.0.0"])] + """ + cleaned_hosts = cleaned["hosts"] + # Get all existing hosts from the database for this domain + existing_hosts_in_db = Host.objects.filter(domain=self) + # Identify hosts to delete + cleaned_host_names = set(cleaned_host["name"] for cleaned_host in cleaned_hosts) + hosts_to_delete_from_db = [ + existing_host for existing_host in existing_hosts_in_db if existing_host.name not in cleaned_host_names + ] + # Delete hosts and their associated HostIP instances + for host_to_delete in hosts_to_delete_from_db: + # Delete associated HostIP instances + HostIP.objects.filter(host=host_to_delete).delete() + # Delete the host itself + host_to_delete.delete() + # Update or create Hosts and HostIPs + for cleaned_host in cleaned_hosts: + # Check if the cleaned_host already exists + host_in_db, host_created = Host.objects.get_or_create(domain=self, name=cleaned_host["name"]) + # Get cleaned list of ips for update + cleaned_ips = cleaned_host["addrs"] + if not host_created: + # Get all existing ips from the database for this host + existing_ips_in_db = HostIP.objects.filter(host=host_in_db) + # Identify IPs to delete + ips_to_delete_from_db = [ + existing_ip for existing_ip in existing_ips_in_db if existing_ip.address not in cleaned_ips + ] + # Delete IPs + for ip_to_delete in ips_to_delete_from_db: + # Delete the ip + ip_to_delete.delete() + # Update or create HostIP instances + for ip_address in cleaned_ips: + HostIP.objects.get_or_create(address=ip_address, host=host_in_db) + def _update_dates(self, cleaned): """Update dates (expiration and creation) from cleaned""" requires_save = False diff --git a/src/registrar/models/domain_application.py b/src/registrar/models/domain_application.py index 95caa41a1..243f029ae 100644 --- a/src/registrar/models/domain_application.py +++ b/src/registrar/models/domain_application.py @@ -745,6 +745,7 @@ class DomainApplication(TimeStampedModel): # Only reject if it exists on EPP if domain_state != Domain.State.UNKNOWN: self.approved_domain.deletedInEpp() + self.approved_domain.save() self.approved_domain.delete() self.approved_domain = None except Exception as err: @@ -783,6 +784,7 @@ class DomainApplication(TimeStampedModel): # Only reject if it exists on EPP if domain_state != Domain.State.UNKNOWN: self.approved_domain.deletedInEpp() + self.approved_domain.save() self.approved_domain.delete() self.approved_domain = None except Exception as err: diff --git a/src/registrar/models/host.py b/src/registrar/models/host.py index bab968afc..2d756111e 100644 --- a/src/registrar/models/host.py +++ b/src/registrar/models/host.py @@ -11,8 +11,8 @@ class Host(TimeStampedModel): The registry is the source of truth for this data. - This model exists ONLY to allow a new registrant to draft DNS entries - before their application is approved. + This model exists to make hosts/nameservers and ip addresses + available when registry is not available. """ name = models.CharField( diff --git a/src/registrar/models/host_ip.py b/src/registrar/models/host_ip.py index 4d646898b..777d14430 100644 --- a/src/registrar/models/host_ip.py +++ b/src/registrar/models/host_ip.py @@ -10,8 +10,8 @@ class HostIP(TimeStampedModel): The registry is the source of truth for this data. - This model exists ONLY to allow a new registrant to draft DNS entries - before their application is approved. + This model exists to make hosts/nameservers and ip addresses + available when registry is not available. """ address = models.CharField( diff --git a/src/registrar/models/nameserver.py b/src/registrar/models/nameserver.py deleted file mode 100644 index 13295f5b5..000000000 --- a/src/registrar/models/nameserver.py +++ /dev/null @@ -1,16 +0,0 @@ -from .host import Host - - -class Nameserver(Host): - """ - A nameserver is a host which has been delegated to respond to DNS queries. - - The registry is the source of truth for this data. - - This model exists ONLY to allow a new registrant to draft DNS entries - before their application is approved. - """ - - # there is nothing here because all of the fields are - # defined over there on the Host class - pass diff --git a/src/registrar/templates/admin/index.html b/src/registrar/templates/admin/index.html new file mode 100644 index 000000000..04601ef32 --- /dev/null +++ b/src/registrar/templates/admin/index.html @@ -0,0 +1,33 @@ +{% extends "admin/index.html" %} + +{% block content %} +
+ {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %} +
+

Reports

+

Domain growth report

+ + {% comment %} + Inputs of type date suck for accessibility. + We'll need to replace those guys with a django form once we figure out how to hook one onto this page. + The challenge is in the path definition in urls. Itdoes NOT like admin/export_data/ + + See the commit "Review for ticket #999" + {% endcomment %} + +
+
+ + +
+
+ + +
+ + +
+ +
+
+{% endblock %} \ No newline at end of file diff --git a/src/registrar/templates/includes/summary_item.html b/src/registrar/templates/includes/summary_item.html index dea14553b..53364d1b2 100644 --- a/src/registrar/templates/includes/summary_item.html +++ b/src/registrar/templates/includes/summary_item.html @@ -47,6 +47,15 @@ {% if value|length == 1 %} {% if users %}

{{ value.0.user.email }}

+ {% elif domains %} + {{ value.0.0 }} + {% if value.0.1 %} + ({% spaceless %} + {% for addr in value.0.1 %} + {{addr}}{% if not forloop.last %}, {% endif %} + {% endfor %} + {% endspaceless %}) + {% endif %} {% else %}

{{ value | first }}

{% endif %} diff --git a/src/registrar/tests/common.py b/src/registrar/tests/common.py index caae195ac..80ec5ef3d 100644 --- a/src/registrar/tests/common.py +++ b/src/registrar/tests/common.py @@ -1184,6 +1184,8 @@ class MockEppLib(TestCase): email=contact.email, voice=contact.voice, fax=contact.fax, + disclose=di, + auth_info=ai, ) def tearDown(self): diff --git a/src/registrar/tests/data/fake_current_federal.csv b/src/registrar/tests/data/fake_current_federal.csv index 33f679e9e..df4fe871f 100644 --- a/src/registrar/tests/data/fake_current_federal.csv +++ b/src/registrar/tests/data/fake_current_federal.csv @@ -1,3 +1,3 @@ -Domain name,Domain type,Agency,Organization name,City,State,Security Contact Email +Domain name,Domain type,Agency,Organization name,City,State,Security contact email cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \ No newline at end of file diff --git a/src/registrar/tests/data/fake_current_full.csv b/src/registrar/tests/data/fake_current_full.csv index 43eefc271..9fef96c60 100644 --- a/src/registrar/tests/data/fake_current_full.csv +++ b/src/registrar/tests/data/fake_current_full.csv @@ -1,4 +1,4 @@ -Domain name,Domain type,Agency,Organization name,City,State,Security Contact Email +Domain name,Domain type,Agency,Organization name,City,State,Security contact email cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, ddomain3.gov,Federal,Armed Forces Retirement Home,,,, adomain2.gov,Interstate,,,,, \ No newline at end of file diff --git a/src/registrar/tests/test_admin_views.py b/src/registrar/tests/test_admin_views.py new file mode 100644 index 000000000..aa150d55c --- /dev/null +++ b/src/registrar/tests/test_admin_views.py @@ -0,0 +1,42 @@ +from django.test import TestCase, Client +from django.urls import reverse +from registrar.tests.common import create_superuser + + +class TestViews(TestCase): + def setUp(self): + self.client = Client(HTTP_HOST="localhost:8080") + self.superuser = create_superuser() + + def test_export_data_view(self): + self.client.force_login(self.superuser) + + # Reverse the URL for the admin index page + admin_index_url = reverse("admin:index") + + # Make a GET request to the admin index page + response = self.client.get(admin_index_url) + + # Assert that the response status code is 200 (OK) + self.assertEqual(response.status_code, 200) + + # Ensure that the start_date and end_date are set + start_date = "2023-01-01" + end_date = "2023-12-31" + + # Construct the URL for the export data view with start_date and end_date parameters: + # This stuff is currently done in JS + export_data_url = reverse("admin_export_data") + f"?start_date={start_date}&end_date={end_date}" + + # Make a GET request to the export data page + response = self.client.get(export_data_url) + + # Assert that the response status code is 200 (OK) or the expected status code + self.assertEqual(response.status_code, 200) + + # Assert that the content type is CSV + self.assertEqual(response["Content-Type"], "text/csv") + + # Check if the filename in the Content-Disposition header matches the expected pattern + expected_filename = f"domain-growth-report-{start_date}-to-{end_date}.csv" + self.assertIn(f'attachment; filename="{expected_filename}"', response["Content-Disposition"]) diff --git a/src/registrar/tests/test_forms.py b/src/registrar/tests/test_forms.py index d54e81427..e0afb2d71 100644 --- a/src/registrar/tests/test_forms.py +++ b/src/registrar/tests/test_forms.py @@ -211,7 +211,7 @@ class TestFormValidation(MockEppLib): def test_other_contact_email_invalid(self): """must be a valid email address.""" - form = OtherContactsForm(data={"email": "boss@boss"}) + form = OtherContactsForm(data={"email": "splendid@boss"}) self.assertEqual( form.errors["email"], ["Enter an email address in the required format, like name@example.com."], @@ -219,7 +219,7 @@ class TestFormValidation(MockEppLib): def test_other_contact_phone_invalid(self): """Must be a valid phone number.""" - form = OtherContactsForm(data={"phone": "boss@boss"}) + form = OtherContactsForm(data={"phone": "super@boss"}) self.assertTrue(form.errors["phone"][0].startswith("Enter a valid phone number ")) def test_requirements_form_blank(self): diff --git a/src/registrar/tests/test_models_domain.py b/src/registrar/tests/test_models_domain.py index 97141c814..c0d4bd27b 100644 --- a/src/registrar/tests/test_models_domain.py +++ b/src/registrar/tests/test_models_domain.py @@ -7,7 +7,7 @@ from django.test import TestCase from django.db.utils import IntegrityError from unittest.mock import MagicMock, patch, call import datetime -from registrar.models import Domain +from registrar.models import Domain, Host, HostIP from unittest import skip from registrar.models.domain_application import DomainApplication @@ -38,6 +38,8 @@ logger = logging.getLogger(__name__) class TestDomainCache(MockEppLib): def tearDown(self): PublicContact.objects.all().delete() + HostIP.objects.all().delete() + Host.objects.all().delete() Domain.objects.all().delete() super().tearDown() @@ -385,6 +387,34 @@ class TestDomainStatuses(MockEppLib): """Domain 'revert_client_hold' method causes the registry to change statuses""" raise + def test_first_ready(self): + """ + first_ready is set when a domain is first transitioned to READY. It does not get overwritten + in case the domain gets out of and back into READY. + """ + domain, _ = Domain.objects.get_or_create(name="pig-knuckles.gov", state=Domain.State.DNS_NEEDED) + self.assertEqual(domain.first_ready, None) + + domain.ready() + + # check that status is READY + self.assertTrue(domain.is_active()) + self.assertNotEqual(domain.first_ready, None) + + # Capture the value of first_ready + first_ready = domain.first_ready + + # change domain status + domain.dns_needed() + self.assertFalse(domain.is_active()) + + # change back to READY + domain.ready() + self.assertTrue(domain.is_active()) + + # assert that the value of first_ready has not changed + self.assertEqual(domain.first_ready, first_ready) + def tearDown(self) -> None: PublicContact.objects.all().delete() Domain.objects.all().delete() @@ -1111,6 +1141,7 @@ class TestRegistrantNameservers(MockEppLib): Then `commands.CreateHost` and `commands.UpdateDomain` is sent to the registry And `domain.is_active` returns False + And domain.first_ready is null """ # set 1 nameserver @@ -1137,6 +1168,8 @@ class TestRegistrantNameservers(MockEppLib): # as you have less than 2 nameservers self.assertFalse(self.domain.is_active()) + self.assertEqual(self.domain.first_ready, None) + def test_user_adds_two_nameservers(self): """ Scenario: Registrant adds 2 or more nameservers, thereby activating the domain @@ -1145,6 +1178,7 @@ class TestRegistrantNameservers(MockEppLib): Then `commands.CreateHost` and `commands.UpdateDomain` is sent to the registry And `domain.is_active` returns True + And domain.first_ready is not null """ # set 2 nameservers @@ -1175,6 +1209,7 @@ class TestRegistrantNameservers(MockEppLib): self.assertEqual(4, self.mockedSendFunction.call_count) # check that status is READY self.assertTrue(self.domain.is_active()) + self.assertNotEqual(self.domain.first_ready, None) def test_user_adds_too_many_nameservers(self): """ @@ -1511,6 +1546,62 @@ class TestRegistrantNameservers(MockEppLib): with self.assertRaises(ActionNotAllowed): domain.nameservers = [self.nameserver1, self.nameserver2] + def test_nameserver_returns_on_registry_error(self): + """ + Scenario: Nameservers previously set through EPP and stored in registrar's database. + Registry is unavailable and throws exception when attempting to build cache from + registry. Nameservers retrieved from database. + """ + domain, _ = Domain.objects.get_or_create(name="fake.gov", state=Domain.State.READY) + # set the host and host_ips directly in the database; this is normally handled through + # fetch_cache + host, _ = Host.objects.get_or_create(domain=domain, name="ns1.fake.gov") + host_ip, _ = HostIP.objects.get_or_create(host=host, address="1.1.1.1") + + # mock that registry throws an error on the InfoHost send + + def side_effect(_request, cleaned): + raise RegistryError(code=ErrorCode.COMMAND_FAILED) + + patcher = patch("registrar.models.domain.registry.send") + mocked_send = patcher.start() + mocked_send.side_effect = side_effect + + nameservers = domain.nameservers + + self.assertEqual(len(nameservers), 1) + self.assertEqual(nameservers[0][0], "ns1.fake.gov") + self.assertEqual(nameservers[0][1], ["1.1.1.1"]) + + patcher.stop() + + def test_nameservers_stored_on_fetch_cache(self): + """ + Scenario: Nameservers are stored in db when they are retrieved from fetch_cache. + Verify the success of this by asserting get_or_create calls to db. + The mocked data for the EPP calls returns a host name + of 'fake.host.com' from InfoDomain and an array of 2 IPs: 1.2.3.4 and 2.3.4.5 + from InfoHost + """ + domain, _ = Domain.objects.get_or_create(name="fake.gov", state=Domain.State.READY) + + # mock the get_or_create methods for Host and HostIP + with patch.object(Host.objects, "get_or_create") as mock_host_get_or_create, patch.object( + HostIP.objects, "get_or_create" + ) as mock_host_ip_get_or_create: + # Set the return value for the mocks + mock_host_get_or_create.return_value = (Host(), True) + mock_host_ip_get_or_create.return_value = (HostIP(), True) + + # force fetch_cache to be called, which will return above documented mocked hosts + domain.nameservers + # assert that the mocks are called + mock_host_get_or_create.assert_called_once_with(domain=domain, name="fake.host.com") + # Retrieve the mocked_host from the return value of the mock + actual_mocked_host, _ = mock_host_get_or_create.return_value + mock_host_ip_get_or_create.assert_called_with(address="2.3.4.5", host=actual_mocked_host) + self.assertEqual(mock_host_ip_get_or_create.call_count, 2) + @skip("not implemented yet") def test_update_is_unsuccessful(self): """ @@ -1529,6 +1620,8 @@ class TestRegistrantNameservers(MockEppLib): domain.nameservers = [("ns1.failednameserver.gov", ["4.5.6"])] def tearDown(self): + HostIP.objects.all().delete() + Host.objects.all().delete() Domain.objects.all().delete() return super().tearDown() @@ -2297,11 +2390,14 @@ class TestAnalystDelete(MockEppLib): When `domain.deletedInEpp()` is called Then `commands.DeleteDomain` is sent to the registry And `state` is set to `DELETED` + + The deleted date is set. """ # Put the domain in client hold self.domain.place_client_hold() # Delete it... self.domain.deletedInEpp() + self.domain.save() self.mockedSendFunction.assert_has_calls( [ call( @@ -2317,6 +2413,9 @@ class TestAnalystDelete(MockEppLib): # Domain should have the right state self.assertEqual(self.domain.state, Domain.State.DELETED) + # Domain should have a deleted + self.assertNotEqual(self.domain.deleted, None) + # Cache should be invalidated self.assertEqual(self.domain._cache, {}) @@ -2335,6 +2434,7 @@ class TestAnalystDelete(MockEppLib): # Delete it with self.assertRaises(RegistryError) as err: domain.deletedInEpp() + domain.save() self.assertTrue(err.is_client_error() and err.code == ErrorCode.OBJECT_ASSOCIATION_PROHIBITS_OPERATION) self.mockedSendFunction.assert_has_calls( [ @@ -2358,12 +2458,18 @@ class TestAnalystDelete(MockEppLib): and domain is of `state` is `READY` Then an FSM error is returned And `state` is not set to `DELETED` + + The deleted date is still null. """ self.assertEqual(self.domain.state, Domain.State.READY) with self.assertRaises(TransitionNotAllowed) as err: self.domain.deletedInEpp() + self.domain.save() self.assertTrue(err.is_client_error() and err.code == ErrorCode.OBJECT_STATUS_PROHIBITS_OPERATION) # Domain should not be deleted self.assertNotEqual(self.domain, None) # Domain should have the right state self.assertEqual(self.domain.state, Domain.State.READY) + + # deleted should be null + self.assertEqual(self.domain.deleted, None) diff --git a/src/registrar/tests/test_reports.py b/src/registrar/tests/test_reports.py index 938f4ed54..e825ec108 100644 --- a/src/registrar/tests/test_reports.py +++ b/src/registrar/tests/test_reports.py @@ -8,7 +8,12 @@ from registrar.models.public_contact import PublicContact from registrar.models.user import User from django.contrib.auth import get_user_model from registrar.tests.common import MockEppLib -from registrar.utility.csv_export import export_domains_to_writer +from registrar.utility.csv_export import ( + write_header, + write_body, + get_default_start_date, + get_default_end_date, +) from django.core.management import call_command from unittest.mock import MagicMock, call, mock_open, patch from api.views import get_current_federal, get_current_full @@ -16,6 +21,8 @@ from django.conf import settings from botocore.exceptions import ClientError import boto3_mocking from registrar.utility.s3_bucket import S3ClientError, S3ClientErrorCodes # type: ignore +from datetime import date, datetime, timedelta +from django.utils import timezone class CsvReportsTest(TestCase): @@ -37,7 +44,6 @@ class CsvReportsTest(TestCase): self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) - self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) self.domain_information_1, _ = DomainInformation.objects.get_or_create( creator=self.user, @@ -77,7 +83,7 @@ class CsvReportsTest(TestCase): mock_client = MagicMock() fake_open = mock_open() expected_file_content = [ - call("Domain name,Domain type,Agency,Organization name,City,State,Security Contact Email\r\n"), + call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), ] @@ -96,7 +102,7 @@ class CsvReportsTest(TestCase): mock_client = MagicMock() fake_open = mock_open() expected_file_content = [ - call("Domain name,Domain type,Agency,Organization name,City,State,Security Contact Email\r\n"), + call("Domain name,Domain type,Agency,Organization name,City,State,Security contact email\r\n"), call("cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,, \r\n"), call("ddomain3.gov,Federal,Armed Forces Retirement Home,,,, \r\n"), call("adomain2.gov,Interstate,,,,, \r\n"), @@ -176,7 +182,7 @@ class CsvReportsTest(TestCase): # Check that the response contains what we expect expected_file_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,Security Contact Email\n" + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" "cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,,,," ).encode() @@ -208,7 +214,7 @@ class CsvReportsTest(TestCase): # Check that the response contains what we expect expected_file_content = ( - "Domain name,Domain type,Agency,Organization name,City,State,Security Contact Email\n" + "Domain name,Domain type,Agency,Organization name,City,State,Security contact email\n" "cdomain1.gov,Federal - Executive,World War I Centennial Commission,,,,\n" "ddomain3.gov,Federal,Armed Forces Retirement Home,,,,\n" "adomain2.gov,Interstate,,,,," @@ -228,10 +234,39 @@ class ExportDataTest(MockEppLib): username=username, first_name=first_name, last_name=last_name, email=email ) - self.domain_1, _ = Domain.objects.get_or_create(name="cdomain1.gov", state=Domain.State.READY) + self.domain_1, _ = Domain.objects.get_or_create( + name="cdomain1.gov", state=Domain.State.READY, first_ready=timezone.now() + ) self.domain_2, _ = Domain.objects.get_or_create(name="adomain2.gov", state=Domain.State.DNS_NEEDED) self.domain_3, _ = Domain.objects.get_or_create(name="ddomain3.gov", state=Domain.State.ON_HOLD) self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_4, _ = Domain.objects.get_or_create(name="bdomain4.gov", state=Domain.State.UNKNOWN) + self.domain_5, _ = Domain.objects.get_or_create( + name="bdomain5.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(2023, 11, 1)) + ) + self.domain_6, _ = Domain.objects.get_or_create( + name="bdomain6.gov", state=Domain.State.DELETED, deleted=timezone.make_aware(datetime(1980, 10, 16)) + ) + self.domain_7, _ = Domain.objects.get_or_create( + name="xdomain7.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + self.domain_8, _ = Domain.objects.get_or_create( + name="sdomain8.gov", state=Domain.State.DELETED, deleted=timezone.now() + ) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + # Deleted yesterday + self.domain_9, _ = Domain.objects.get_or_create( + name="zdomain9.gov", + state=Domain.State.DELETED, + deleted=timezone.make_aware(datetime.combine(date.today() - timedelta(days=1), datetime.min.time())), + ) + # ready tomorrow + self.domain_10, _ = Domain.objects.get_or_create( + name="adomain10.gov", + state=Domain.State.READY, + first_ready=timezone.make_aware(datetime.combine(date.today() + timedelta(days=1), datetime.min.time())), + ) self.domain_information_1, _ = DomainInformation.objects.get_or_create( creator=self.user, @@ -257,6 +292,42 @@ class ExportDataTest(MockEppLib): organization_type="federal", federal_agency="Armed Forces Retirement Home", ) + self.domain_information_5, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_5, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) + self.domain_information_6, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_6, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) + self.domain_information_7, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_7, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) + self.domain_information_8, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_8, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) + self.domain_information_9, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_9, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) + self.domain_information_10, _ = DomainInformation.objects.get_or_create( + creator=self.user, + domain=self.domain_10, + organization_type="federal", + federal_agency="Armed Forces Retirement Home", + ) def tearDown(self): PublicContact.objects.all().delete() @@ -265,8 +336,8 @@ class ExportDataTest(MockEppLib): User.objects.all().delete() super().tearDown() - def test_export_domains_to_writer(self): - """Test that export_domains_to_writer returns the + def test_write_body(self): + """Test that write_body returns the existing domain, test that sort by domain name works, test that filter works""" # Create a CSV file in memory @@ -287,7 +358,7 @@ class ExportDataTest(MockEppLib): "Submitter title", "Submitter email", "Submitter phone", - "Security Contact Email", + "Security contact email", "Status", ] sort_fields = ["domain__name"] @@ -299,8 +370,9 @@ class ExportDataTest(MockEppLib): ], } - # Call the export function - export_domains_to_writer(writer, columns, sort_fields, filter_condition) + # Call the export functions + write_header(writer, columns) + write_body(writer, columns, sort_fields, filter_condition) # Reset the CSV file's position to the beginning csv_file.seek(0) @@ -313,10 +385,11 @@ class ExportDataTest(MockEppLib): expected_content = ( "Domain name,Domain type,Agency,Organization name,City,State,AO," "AO email,Submitter,Submitter title,Submitter email,Submitter phone," - "Security Contact Email,Status\n" - "adomain2.gov,Interstate,dnsneeded\n" - "cdomain1.gov,Federal - Executive,World War I Centennial Commission,ready\n" - "ddomain3.gov,Federal,Armed Forces Retirement Home,onhold\n" + "Security contact email,Status\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,Ready\n" + "adomain2.gov,Interstate,Dns needed\n" + "cdomain1.gov,Federal - Executive,World War I Centennial Commission,Ready\n" + "ddomain3.gov,Federal,Armed Forces Retirement Home,On hold\n" ) # Normalize line endings and remove commas, @@ -399,7 +472,7 @@ class ExportDataTest(MockEppLib): self.assertEqual(csv_content, expected_content) - def test_export_domains_to_writer_additional(self): + def test_write_body_additional(self): """An additional test for filters and multi-column sort""" # Create a CSV file in memory csv_file = StringIO() @@ -413,7 +486,7 @@ class ExportDataTest(MockEppLib): "Organization name", "City", "State", - "Security Contact Email", + "Security contact email", ] sort_fields = ["domain__name", "federal_agency", "organization_type"] filter_condition = { @@ -425,8 +498,9 @@ class ExportDataTest(MockEppLib): ], } - # Call the export function - export_domains_to_writer(writer, columns, sort_fields, filter_condition) + # Call the export functions + write_header(writer, columns) + write_body(writer, columns, sort_fields, filter_condition) # Reset the CSV file's position to the beginning csv_file.seek(0) @@ -439,7 +513,8 @@ class ExportDataTest(MockEppLib): # sorted alphabetially by domain name expected_content = ( "Domain name,Domain type,Agency,Organization name,City," - "State,Security Contact Email\n" + "State,Security contact email\n" + "adomain10.gov,Federal,Armed Forces Retirement Home\n" "cdomain1.gov,Federal - Executive,World War I Centennial Commission\n" "ddomain3.gov,Federal,Armed Forces Retirement Home\n" ) @@ -450,3 +525,113 @@ class ExportDataTest(MockEppLib): expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() self.assertEqual(csv_content, expected_content) + + def test_write_body_with_date_filter_pulls_domains_in_range(self): + """Test that domains that are + 1. READY and their first_ready dates are in range + 2. DELETED and their deleted dates are in range + are pulled when the growth report conditions are applied to export_domains_to_writed. + Test that ready domains are sorted by first_ready/deleted dates first, names second. + + We considered testing export_data_growth_to_csv which calls write_body + and would have been easy to set up, but expected_content would contain created_at dates + which are hard to mock. + + TODO: Simplify is created_at is not needed for the report.""" + + # Create a CSV file in memory + csv_file = StringIO() + writer = csv.writer(csv_file) + # We use timezone.make_aware to sync to server time a datetime object with the current date (using date.today()) + # and a specific time (using datetime.min.time()). + end_date = timezone.make_aware(datetime.combine(date.today() + timedelta(days=2), datetime.min.time())) + start_date = timezone.make_aware(datetime.combine(date.today() - timedelta(days=2), datetime.min.time())) + + # Define columns, sort fields, and filter condition + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + ] + sort_fields = [ + "created_at", + "domain__name", + ] + sort_fields_for_deleted_domains = [ + "domain__deleted", + "domain__name", + ] + filter_condition = { + "domain__state__in": [ + Domain.State.READY, + ], + "domain__first_ready__lte": end_date, + "domain__first_ready__gte": start_date, + } + filter_conditions_for_deleted_domains = { + "domain__state__in": [ + Domain.State.DELETED, + ], + "domain__deleted__lte": end_date, + "domain__deleted__gte": start_date, + } + + # Call the export functions + write_header(writer, columns) + write_body( + writer, + columns, + sort_fields, + filter_condition, + ) + write_body( + writer, + columns, + sort_fields_for_deleted_domains, + filter_conditions_for_deleted_domains, + ) + + # Reset the CSV file's position to the beginning + csv_file.seek(0) + + # Read the content into a variable + csv_content = csv_file.read() + + # We expect READY domains first, created between today-2 and today+2, sorted by created_at then name + # and DELETED domains deleted between today-2 and today+2, sorted by deleted then name + expected_content = ( + "Domain name,Domain type,Agency,Organization name,City," + "State,Status,Expiration date\n" + "cdomain1.gov,Federal-Executive,World War I Centennial Commission,,,,Ready,\n" + "adomain10.gov,Federal,Armed Forces Retirement Home,,,,Ready,\n" + "zdomain9.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n" + "sdomain8.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n" + "xdomain7.gov,Federal,Armed Forces Retirement Home,,,,Deleted,\n" + ) + + # Normalize line endings and remove commas, + # spaces and leading/trailing whitespace + csv_content = csv_content.replace(",,", "").replace(",", "").replace(" ", "").replace("\r\n", "\n").strip() + expected_content = expected_content.replace(",,", "").replace(",", "").replace(" ", "").strip() + + self.assertEqual(csv_content, expected_content) + + +class HelperFunctions(TestCase): + """This asserts that 1=1. Its limited usefulness lies in making sure the helper methods stay healthy.""" + + def test_get_default_start_date(self): + expected_date = timezone.make_aware(datetime(2023, 11, 1)) + actual_date = get_default_start_date() + self.assertEqual(actual_date, expected_date) + + def test_get_default_end_date(self): + # Note: You may need to mock timezone.now() for accurate testing + expected_date = timezone.now() + actual_date = get_default_end_date() + self.assertEqual(actual_date.date(), expected_date.date()) diff --git a/src/registrar/tests/test_views.py b/src/registrar/tests/test_views.py index fe8d87256..a195f5f1a 100644 --- a/src/registrar/tests/test_views.py +++ b/src/registrar/tests/test_views.py @@ -28,6 +28,8 @@ from registrar.models import ( DomainInvitation, Contact, PublicContact, + Host, + HostIP, Website, UserDomainRole, User, @@ -725,6 +727,92 @@ class DomainApplicationTests(TestWithUser, WebTest): actual_url_slug = no_contacts_page.request.path.split("/")[-2] self.assertEqual(expected_url_slug, actual_url_slug) + def test_application_delete_other_contact(self): + """Other contacts can be deleted after being saved to database.""" + # Populate the databse with a domain application that + # has 1 "other contact" assigned to it + ao, _ = Contact.objects.get_or_create( + first_name="Testy", + last_name="Tester", + title="Chief Tester", + email="testy@town.com", + phone="(555) 555 5555", + ) + you, _ = Contact.objects.get_or_create( + first_name="Testy you", + last_name="Tester you", + title="Admin Tester", + email="testy-admin@town.com", + phone="(555) 555 5556", + ) + other, _ = Contact.objects.get_or_create( + first_name="Testy2", + last_name="Tester2", + title="Another Tester", + email="testy2@town.com", + phone="(555) 555 5557", + ) + application, _ = DomainApplication.objects.get_or_create( + organization_type="federal", + federal_type="executive", + purpose="Purpose of the site", + anything_else="No", + is_policy_acknowledged=True, + organization_name="Testorg", + address_line1="address 1", + state_territory="NY", + zipcode="10002", + authorizing_official=ao, + submitter=you, + creator=self.user, + status="started", + ) + application.other_contacts.add(other) + + # prime the form by visiting /edit + self.app.get(reverse("edit-application", kwargs={"id": application.pk})) + # django-webtest does not handle cookie-based sessions well because it keeps + # resetting the session key on each new request, thus destroying the concept + # of a "session". We are going to do it manually, saving the session ID here + # and then setting the cookie on each request. + session_id = self.app.cookies[settings.SESSION_COOKIE_NAME] + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + other_contacts_page = self.app.get(reverse("application:other_contacts")) + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + other_contacts_form = other_contacts_page.forms[0] + + # Minimal check to ensure the form is loaded with data (if this part of + # the application doesn't work, we should be equipped with other unit + # tests to flag it) + self.assertEqual(other_contacts_form["other_contacts-0-first_name"].value, "Testy2") + + # clear the form + other_contacts_form["other_contacts-0-first_name"] = "" + other_contacts_form["other_contacts-0-middle_name"] = "" + other_contacts_form["other_contacts-0-last_name"] = "" + other_contacts_form["other_contacts-0-title"] = "" + other_contacts_form["other_contacts-0-email"] = "" + other_contacts_form["other_contacts-0-phone"] = "" + + # Submit the now empty form + result = other_contacts_form.submit() + self.app.set_cookie(settings.SESSION_COOKIE_NAME, session_id) + + # Verify that the contact we saved earlier has been removed from the database + application = DomainApplication.objects.get() # There are no contacts anymore + self.assertEqual( + application.other_contacts.count(), + 0, + ) + + # Verify that on submit, user is advanced to "no contacts" page + no_contacts_page = result.follow() + expected_url_slug = str(Step.NO_OTHER_CONTACTS) + actual_url_slug = no_contacts_page.request.path.split("/")[-2] + self.assertEqual(expected_url_slug, actual_url_slug) + def test_application_about_your_organiztion_interstate(self): """Special districts have to answer an additional question.""" type_page = self.app.get(reverse("application:")).follow() @@ -1173,6 +1261,8 @@ class TestWithDomainPermissions(TestWithUser): DomainApplication.objects.all().delete() DomainInformation.objects.all().delete() PublicContact.objects.all().delete() + HostIP.objects.all().delete() + Host.objects.all().delete() Domain.objects.all().delete() UserDomainRole.objects.all().delete() except ValueError: # pass if already deleted diff --git a/src/registrar/utility/csv_export.py b/src/registrar/utility/csv_export.py index 8e9e40b2a..e3de75dec 100644 --- a/src/registrar/utility/csv_export.py +++ b/src/registrar/utility/csv_export.py @@ -1,54 +1,93 @@ import csv +import logging +from datetime import datetime from registrar.models.domain import Domain from registrar.models.domain_information import DomainInformation from registrar.models.public_contact import PublicContact from django.db.models import Value from django.db.models.functions import Coalesce +from django.utils import timezone + +logger = logging.getLogger(__name__) -def export_domains_to_writer(writer, columns, sort_fields, filter_condition): - # write columns headers to writer +def write_header(writer, columns): + """ + Receives params from the parent methods and outputs a CSV with a header row. + Works with write_header as longas the same writer object is passed. + """ writer.writerow(columns) - domainInfos = DomainInformation.objects.filter(**filter_condition).order_by(*sort_fields) - for domainInfo in domainInfos: - security_contacts = domainInfo.domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) - # For linter - ao = " " - if domainInfo.authorizing_official: - first_name = domainInfo.authorizing_official.first_name or "" - last_name = domainInfo.authorizing_official.last_name or "" - ao = first_name + " " + last_name - security_email = " " - if security_contacts: - security_email = security_contacts[0].email +def get_domain_infos(filter_condition, sort_fields): + domain_infos = DomainInformation.objects.filter(**filter_condition).order_by(*sort_fields) + return domain_infos - invalid_emails = {"registrar@dotgov.gov", "dotgov@cisa.dhs.gov"} - # These are default emails that should not be displayed in the csv report - if security_email is not None and security_email.lower() in invalid_emails: - security_email = "(blank)" - # create a dictionary of fields which can be included in output - FIELDS = { - "Domain name": domainInfo.domain.name, - "Domain type": domainInfo.get_organization_type_display() + " - " + domainInfo.get_federal_type_display() - if domainInfo.federal_type - else domainInfo.get_organization_type_display(), - "Agency": domainInfo.federal_agency, - "Organization name": domainInfo.organization_name, - "City": domainInfo.city, - "State": domainInfo.state_territory, - "AO": ao, - "AO email": domainInfo.authorizing_official.email if domainInfo.authorizing_official else " ", - "Security Contact Email": security_email, - "Status": domainInfo.domain.state, - "Expiration Date": domainInfo.domain.expiration_date, - } - writer.writerow([FIELDS.get(column, "") for column in columns]) +def write_row(writer, columns, domain_info): + security_contacts = domain_info.domain.contacts.filter(contact_type=PublicContact.ContactTypeChoices.SECURITY) + # For linter + ao = " " + if domain_info.authorizing_official: + first_name = domain_info.authorizing_official.first_name or "" + last_name = domain_info.authorizing_official.last_name or "" + ao = first_name + " " + last_name + + security_email = " " + if security_contacts: + security_email = security_contacts[0].email + + invalid_emails = {"registrar@dotgov.gov", "dotgov@cisa.dhs.gov"} + # These are default emails that should not be displayed in the csv report + if security_email is not None and security_email.lower() in invalid_emails: + security_email = "(blank)" + + # create a dictionary of fields which can be included in output + FIELDS = { + "Domain name": domain_info.domain.name, + "Domain type": domain_info.get_organization_type_display() + " - " + domain_info.get_federal_type_display() + if domain_info.federal_type + else domain_info.get_organization_type_display(), + "Agency": domain_info.federal_agency, + "Organization name": domain_info.organization_name, + "City": domain_info.city, + "State": domain_info.state_territory, + "AO": ao, + "AO email": domain_info.authorizing_official.email if domain_info.authorizing_official else " ", + "Security contact email": security_email, + "Status": domain_info.domain.get_state_display(), + "Expiration date": domain_info.domain.expiration_date, + "Created at": domain_info.domain.created_at, + "First ready": domain_info.domain.first_ready, + "Deleted": domain_info.domain.deleted, + } + writer.writerow([FIELDS.get(column, "") for column in columns]) + + +def write_body( + writer, + columns, + sort_fields, + filter_condition, +): + """ + Receives params from the parent methods and outputs a CSV with fltered and sorted domains. + Works with write_header as longas the same writer object is passed. + """ + + # Get the domainInfos + domain_infos = get_domain_infos(filter_condition, sort_fields) + + all_domain_infos = list(domain_infos) + + # Write rows to CSV + for domain_info in all_domain_infos: + write_row(writer, columns, domain_info) def export_data_type_to_csv(csv_file): + """All domains report with extra columns""" + writer = csv.writer(csv_file) # define columns to include in export columns = [ @@ -60,9 +99,9 @@ def export_data_type_to_csv(csv_file): "State", "AO", "AO email", - "Security Contact Email", + "Security contact email", "Status", - "Expiration Date", + "Expiration date", ] # Coalesce is used to replace federal_type of None with ZZZZZ sort_fields = [ @@ -78,10 +117,13 @@ def export_data_type_to_csv(csv_file): Domain.State.ON_HOLD, ], } - export_domains_to_writer(writer, columns, sort_fields, filter_condition) + write_header(writer, columns) + write_body(writer, columns, sort_fields, filter_condition) def export_data_full_to_csv(csv_file): + """All domains report""" + writer = csv.writer(csv_file) # define columns to include in export columns = [ @@ -91,7 +133,7 @@ def export_data_full_to_csv(csv_file): "Organization name", "City", "State", - "Security Contact Email", + "Security contact email", ] # Coalesce is used to replace federal_type of None with ZZZZZ sort_fields = [ @@ -107,10 +149,13 @@ def export_data_full_to_csv(csv_file): Domain.State.ON_HOLD, ], } - export_domains_to_writer(writer, columns, sort_fields, filter_condition) + write_header(writer, columns) + write_body(writer, columns, sort_fields, filter_condition) def export_data_federal_to_csv(csv_file): + """Federal domains report""" + writer = csv.writer(csv_file) # define columns to include in export columns = [ @@ -120,7 +165,7 @@ def export_data_federal_to_csv(csv_file): "Organization name", "City", "State", - "Security Contact Email", + "Security contact email", ] # Coalesce is used to replace federal_type of None with ZZZZZ sort_fields = [ @@ -137,4 +182,74 @@ def export_data_federal_to_csv(csv_file): Domain.State.ON_HOLD, ], } - export_domains_to_writer(writer, columns, sort_fields, filter_condition) + write_header(writer, columns) + write_body(writer, columns, sort_fields, filter_condition) + + +def get_default_start_date(): + # Default to a date that's prior to our first deployment + return timezone.make_aware(datetime(2023, 11, 1)) + + +def get_default_end_date(): + # Default to now() + return timezone.now() + + +def export_data_growth_to_csv(csv_file, start_date, end_date): + """ + Growth report: + Receive start and end dates from the view, parse them. + Request from write_body READY domains that are created between + the start and end dates, as well as DELETED domains that are deleted between + the start and end dates. Specify sort params for both lists. + """ + + start_date_formatted = ( + timezone.make_aware(datetime.strptime(start_date, "%Y-%m-%d")) if start_date else get_default_start_date() + ) + + end_date_formatted = ( + timezone.make_aware(datetime.strptime(end_date, "%Y-%m-%d")) if end_date else get_default_end_date() + ) + + writer = csv.writer(csv_file) + + # define columns to include in export + columns = [ + "Domain name", + "Domain type", + "Agency", + "Organization name", + "City", + "State", + "Status", + "Expiration date", + "Created at", + "First ready", + "Deleted", + ] + sort_fields = [ + "domain__first_ready", + "domain__name", + ] + filter_condition = { + "domain__state__in": [Domain.State.READY], + "domain__first_ready__lte": end_date_formatted, + "domain__first_ready__gte": start_date_formatted, + } + + # We also want domains deleted between sar and end dates, sorted + sort_fields_for_deleted_domains = [ + "domain__deleted", + "domain__name", + ] + filter_condition_for_deleted_domains = { + "domain__state__in": [Domain.State.DELETED], + "domain__deleted__lte": end_date_formatted, + "domain__deleted__gte": start_date_formatted, + } + + write_header(writer, columns) + write_body(writer, columns, sort_fields, filter_condition) + write_body(writer, columns, sort_fields_for_deleted_domains, filter_condition_for_deleted_domains) diff --git a/src/registrar/views/admin_views.py b/src/registrar/views/admin_views.py new file mode 100644 index 000000000..f7164663b --- /dev/null +++ b/src/registrar/views/admin_views.py @@ -0,0 +1,26 @@ +"""Admin-related views.""" + +from django.http import HttpResponse +from django.views import View + +from registrar.utility import csv_export + +import logging + +logger = logging.getLogger(__name__) + + +class ExportData(View): + def get(self, request, *args, **kwargs): + # Get start_date and end_date from the request's GET parameters + # #999: not needed if we switch to django forms + start_date = request.GET.get("start_date", "") + end_date = request.GET.get("end_date", "") + + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = f'attachment; filename="domain-growth-report-{start_date}-to-{end_date}.csv"' + # For #999: set export_data_growth_to_csv to return the resulting queryset, which we can then use + # in context to display this data in the template. + csv_export.export_data_growth_to_csv(response, start_date, end_date) + + return response