Resolve merge conflict

.github/workflows/test.yaml

@@ -43,9 +43,15 @@ jobs:
         run: |
           perl -pi \
           -e 's/"csp.middleware.CSPMiddleware",/$&"registrar.tests.common.MockUserLogin",/' \
-          src/registrar/config/settings.py
+          registrar/config/settings.py
 
-      - name: Accessibility Scan
+      - name: Start container
         working-directory: ./src
         # leverage the docker compose setup that we already have for local development
-        run: docker compose run pa11y npm run pa11y-ci
+        run: docker compose up -d 
+
+      - name: run pa11y
+        working-directory: ./src
+        run: |
+          npm i -g pa11y-ci
+          pa11y-ci 

docs/operations/data_migration.md

@@ -0,0 +1,79 @@
+# Registrar Data Migration
+
+There is an existing registrar/registry at Verisign. They will provide us with an
+export of the data from that system. The goal of our data migration is to take
+the provided data and use it to create as much as possible a _matching_ state
+in our registrar.
+
+There is no way to make our registrar _identical_ to the Verisign system
+because we have a different data model and workflow model. Instead, we should
+focus our migration efforts on creating a state in our new registrar that will
+primarily allow users of the system to perform the tasks that they want to do.
+
+## Users
+
+One of the major differences with the existing registrar/registry is that our
+system uses Login.gov for authentication. Any person with an identity-verified
+Login.gov account can make an account on the new registrar, and the first time
+that person logs in through Login.gov, we make a corresponding account in our
+user table. Because we cannot know the Universal Unique ID (UUID) for a
+person's Login.gov account, we cannot pre-create user accounts for individuals
+in our new registrar based on the data from Verisign.
+
+## Domains
+
+Our registrar keeps track of domains. The authoritative source for domain
+information is the registry, but the registrar needs a copy of that
+information to make connections between registry users and the domains that
+they manage. The registrar stores very few fields about a domain except for
+its name, so it could be straightforward to import the exported list of domains
+from Verisign's `escrow_domains.daily.dotgov.GOV.txt`. It doesn't appear that
+that table stores a flag for active or inactive, so every domain in the file
+can be imported into our system as `is_active=True`.
+
+An example Django management command that can load the delimited text file
+from the daily escrow is in
+`src/registrar/management/commands/load_domains_data.py`. It uses Django's
+object-relational modeler (ORM) to create Django objects for the domains and
+then write them to the database in a single bulk operation. To run the command
+locally for testing, using Docker Compose:
+
+```shell
+docker compose run -T app ./manage.py load_domains_data < /tmp/escrow_domains.daily.dotgov.GOV.txt
+```
+
+## User access to domains
+
+The Verisign data contains a `escrow_domain_contacts.daily.dotgov.txt` file
+that links each domain to three different types of contacts: `billing`,
+`tech`, and `admin`. The ID of the contact in this linking table corresponds
+to the ID of a contact in the `escrow_contacts.daily.dotgov.txt` file. In the
+contacts file is an email address for each contact.
+
+The new registrar associates user accounts (authenticated with Login.gov) with
+domains using a `UserDomainRole` linking table. New users can be granted roles
+on domains by creating a `DomainInvitation` that links an email address with a
+domain. When a new user finishes authenticating with Login.gov and their email
+address matches an invitation, then they are given the appropriate role on the
+invitation's domain.
+
+For the purposes of migration, we can prime the invitation system by creating
+an invitation in the system for each email address listed in the
+`domain_contacts` file. This means that if a person is currently a user in the
+Verisign system, and they use the same email address with Login.gov, then they
+will end up with access to the same domains in the new registrar that they
+were associated with in the Verisign system.
+
+A management command that does this needs to process two data files, one for
+the contact information and one for the domain/contact association, so we
+can't use stdin the way that we did before. Instead, we can use the fact that
+Docker Compose mounts the `src/` directory inside of the container at `/app`.
+Then, data files that are inside of the `src/` directory can be accessed
+inside the Docker container.
+
+An example script using this technique is in
+`src/registrar/management/commands/load_domain_invitations.py`.
+
+```shell
+docker compose run app ./manage.py load_domain_invitations /app/escrow_domain_contacts.daily.dotgov.GOV.txt /app/escrow_contacts.daily.dotgov.GOV.txt
+```

src/.pa11yci

@@ -1,22 +1,28 @@
 {
+    "defaults": {
+        "concurrency": 1,
+        "timeout": 10000,
+        "hideElements": "a[href='/whoami/']"
+
+    },
     "urls": [
-        "http://app:8080/",
+        "http://localhost:8080/",
-        "http://app:8080/health/",
+        "http://localhost:8080/health/",
-        "http://app:8080/whoami/",
+        "http://localhost:8080/whoami/",
-        "http://app:8080/register/",
+        "http://localhost:8080/register/",
-        "http://app:8080/register/organization/",
+        "http://localhost:8080/register/organization/",
-        "http://app:8080/register/org_federal/",
+        "http://localhost:8080/register/org_federal/",
-        "http://app:8080/register/org_election/",
+        "http://localhost:8080/register/org_election/",
-        "http://app:8080/register/org_contact/",
+        "http://localhost:8080/register/org_contact/",
-        "http://app:8080/register/authorizing_official/",
+        "http://localhost:8080/register/authorizing_official/",
-        "http://app:8080/register/current_sites/",
+        "http://localhost:8080/register/current_sites/",
-        "http://app:8080/register/dotgov_domain/",
+        "http://localhost:8080/register/dotgov_domain/",
-        "http://app:8080/register/purpose/",
+        "http://localhost:8080/register/purpose/",
-        "http://app:8080/register/your_contact/",
+        "http://localhost:8080/register/your_contact/",
-        "http://app:8080/register/other_contacts/",
+        "http://localhost:8080/register/other_contacts/",
-        "http://app:8080/register/anything_else/",
+        "http://localhost:8080/register/anything_else/",
-        "http://app:8080/register/requirements/",
+        "http://localhost:8080/register/requirements/",
-        "http://app:8080/register/review/",
+        "http://localhost:8080/register/review/",
-        "http://app:8080/register/finished/"
+        "http://localhost:8080/register/finished/"
     ]
 }

src/registrar/assets/sass/_theme/_uswds-theme-custom-styles.scss

@@ -133,11 +133,47 @@ a.breadcrumb__back {
   }
 }
 
-.withdraw_outline {
+a.withdraw_outline {
   box-shadow: inset 0 0 0 2px color('error');
   color: color('error');
 }
 
+a.withdraw_outline:visited {
+  box-shadow: inset 0 0 0 2px color('error');
+  color: color('error');
+}
+
+a.withdraw_outline:hover {
+  box-shadow: inset 0 0 0 2px color('error');
+  color: color('error-dark');
+}
+
+a.withdraw_outline:focus {
+  box-shadow: inset 0 0 0 2px color('error');
+  color: color('error-dark');
+}
+
+a.withdraw_outline:active {
+  box-shadow: inset 0 0 0 2px color('error');
+  color: color('error-darker');
+}
+
+a.withdraw:focus {
+  background-color: color('error-dark');
+}
+
+a.withdraw:hover {
+  background-color: color('error-dark');
+}
+
+a.withdraw:active {
+  background-color: color('error-darker');
+}
+
+a.withdraw {
+  background-color: color('error');
+}
+
 .usa-sidenav {
   .usa-sidenav__item {
     span {

src/registrar/config/urls.py

@@ -57,6 +57,16 @@ urlpatterns = [
         views.ApplicationStatus.as_view(),
         name="application-status",
     ),
+    path(
+        "application/<int:pk>/withdraw",
+        views.ApplicationWithdraw.as_view(),
+        name="application-withdraw-confirmation",
+    ),
+    path(
+        "application/<int:pk>/withdrawconfirmed",
+        views.ApplicationWithdraw.updatestatus,
+        name="application-withdrawn",
+    ),
     path("health/", views.health),
     path("openid/", include("djangooidc.urls")),
     path("register/", include((application_urls, APPLICATION_NAMESPACE))),

src/registrar/fixtures.py

@@ -116,6 +116,10 @@ class DomainApplicationFixture:
             "status": "investigating",
             "organization_name": "Example - Approved",
         },
+        {
+            "status": "withdrawn",
+            "organization_name": "Example - Withdrawn",
+        },
     ]
 
     @classmethod

src/registrar/management/commands/load_domain_invitations.py

@@ -0,0 +1,76 @@
+"""Load domain invitations for existing domains and their contacts."""
+
+import csv
+import logging
+
+from collections import defaultdict
+
+from django.core.management import BaseCommand
+
+from registrar.models import Domain, DomainInvitation
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Load invitations for existing domains and their users."
+
+    def add_arguments(self, parser):
+        """Add our two filename arguments."""
+        parser.add_argument(
+            "domain_contacts_filename",
+            help="Data file with domain contact information",
+        )
+        parser.add_argument(
+            "contacts_filename", help="Data file with contact information"
+        )
+
+        parser.add_argument("--sep", default="|", help="Delimiter character")
+
+    def handle(self, domain_contacts_filename, contacts_filename, **options):
+        """Load the data files and create the DomainInvitations."""
+        sep = options.get("sep")
+
+        # We open the domain file first and hold it in memory.
+        # There are three contacts per domain, so there should be at
+        # most 3*N different contacts here.
+        contact_domains = defaultdict(list)  # each contact has a list of domains
+        logger.info("Reading domain-contacts data file %s", domain_contacts_filename)
+        with open(domain_contacts_filename, "r") as domain_file:
+            for row in csv.reader(domain_file, delimiter=sep):
+                # fields are just domain, userid, role
+                # lowercase the domain names now
+                contact_domains[row[1]].append(row[0].lower())
+        logger.info("Loaded domains for %d contacts", len(contact_domains))
+
+        # now we have a mapping of user IDs to lists of domains for that user
+        # iterate over the contacts list and for contacts in our mapping,
+        # create the domain invitations for their email address
+        logger.info("Reading contacts data file %s", contacts_filename)
+        to_create = []
+        skipped = 0
+        with open(contacts_filename, "r") as contacts_file:
+            for row in csv.reader(contacts_file, delimiter=sep):
+                # userid is in the first field, email is the seventh
+                userid = row[0]
+                if userid not in contact_domains:
+                    # this user has no domains, skip them
+                    skipped += 1
+                    continue
+                for domain_name in contact_domains[userid]:
+                    email_address = row[6]
+                    domain = Domain.objects.get(name=domain_name)
+                    to_create.append(
+                        DomainInvitation(
+                            email=email_address.lower(),
+                            domain=domain,
+                            status=DomainInvitation.INVITED,
+                        )
+                    )
+        logger.info("Creating %d invitations", len(to_create))
+        DomainInvitation.objects.bulk_create(to_create)
+        logger.info(
+            "Created %d domain invitations, ignored %d contacts",
+            len(to_create),
+            skipped,
+        )

src/registrar/management/commands/load_domains_data.py

@@ -0,0 +1,69 @@
+"""Load domains from registry export."""
+
+import csv
+import logging
+import sys
+
+from django.core.management.base import BaseCommand
+
+from registrar.models import Domain
+
+
+logger = logging.getLogger(__name__)
+
+
+def _domain_dict_reader(file_object, **kwargs):
+    """A csv DictReader with the correct field names for escrow_domains data.
+
+    All keyword arguments are sent on to the DictReader function call.
+    """
+    # field names are from escrow_manifests without "f"
+    return csv.DictReader(
+        file_object,
+        fieldnames=[
+            "Name",
+            "Roid",
+            "IdnTableId",
+            "Registrant",
+            "ClID",
+            "CrRr",
+            "CrID",
+            "CrDate",
+            "UpRr",
+            "UpID",
+            "UpDate",
+            "ExDate",
+            "TrDate",
+        ],
+        **kwargs,
+    )
+
+
+class Command(BaseCommand):
+    help = "Load domain data from a delimited text file on stdin."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--sep", default="|", help="Separator character for data file"
+        )
+
+    def handle(self, *args, **options):
+        separator_character = options.get("sep")
+        reader = _domain_dict_reader(sys.stdin, delimiter=separator_character)
+        # accumulate model objects so we can `bulk_create` them all at once.
+        domains = []
+        for row in reader:
+            name = row["Name"].lower()  # we typically use lowercase domains
+
+            # Ensure that there is a `Domain` object for each domain name in
+            # this file and that it is active. There is a uniqueness
+            # constraint for active Domain objects, so we are going to account
+            # for that here with this check so that our later bulk_create
+            # should succeed
+            if Domain.objects.filter(name=name, is_active=True).exists():
+                # don't do anything, this domain is here and active
+                continue
+            else:
+                domains.append(Domain(name=name, is_active=True))
+        logger.info("Creating %d new domains", len(domains))
+        Domain.objects.bulk_create(domains)

src/registrar/migrations/0017_alter_domainapplication_status_and_more.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.1.6 on 2023-04-13 18:38
+
+from django.db import migrations
+import django_fsm  # type: ignore
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("registrar", "0016_domaininvitation"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="domainapplication",
+            name="status",
+            field=django_fsm.FSMField(
+                choices=[
+                    ("started", "started"),
+                    ("submitted", "submitted"),
+                    ("investigating", "investigating"),
+                    ("approved", "approved"),
+                    ("withdrawn", "withdrawn"),
+                ],
+                default="started",
+                max_length=50,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="domaininvitation",
+            name="status",
+            field=django_fsm.FSMField(
+                choices=[("invited", "invited"), ("retrieved", "retrieved")],
+                default="invited",
+                max_length=50,
+                protected=True,
+            ),
+        ),
+    ]

src/registrar/models/domain_application.py

@@ -23,11 +23,13 @@ class DomainApplication(TimeStampedModel):
     SUBMITTED = "submitted"
     INVESTIGATING = "investigating"
     APPROVED = "approved"
+    WITHDRAWN = "withdrawn"
     STATUS_CHOICES = [
         (STARTED, STARTED),
         (SUBMITTED, SUBMITTED),
         (INVESTIGATING, INVESTIGATING),
         (APPROVED, APPROVED),
+        (WITHDRAWN, WITHDRAWN),
     ]
 
     class StateTerritoryChoices(models.TextChoices):
@@ -483,7 +485,7 @@ class DomainApplication(TimeStampedModel):
         except EmailSendingError:
             logger.warning("Failed to send confirmation email", exc_info=True)
 
-    @transition(field="status", source=STARTED, target=SUBMITTED)
+    @transition(field="status", source=[STARTED, WITHDRAWN], target=SUBMITTED)
     def submit(self):
         """Submit an application that is started."""
 

src/registrar/templates/application_authorizing_official.html

@@ -7,7 +7,7 @@
   </h2>
 
   <p>Your authorizing official is the person within your organization who can authorize
-  your domain request. This is generally the highest ranking or highest elected official
+  your domain request. This is generally the highest-ranking or highest-elected official
   in your organization. Read more about <a href="{% url 'todo' %}">who can serve as an
   authorizing official</a>.</p>
 
@@ -41,4 +41,4 @@
     {% endwith %}
 
   </fieldset>
-{% endblock %}
+{% endblock %}

src/registrar/templates/application_status.html

@@ -1,6 +1,6 @@
 {% extends 'base.html' %}
 
-{% block title %}Domain request status- {{ domainapplication.requested_domain.name }}{% endblock %}
+{% block title %}Domain request status- {{ domainapplication.requested_domain.name }} | {% endblock %}
 {% load static url_helpers %}
 
 {% block content %}
@@ -31,8 +31,8 @@
     <p> <b class="review__step__name">Last updated:</b> {{domainapplication.updated_at|date:"F j, Y"}}<br>
     <b class="review__step__name">Request #:</b> {{domainapplication.id}}</p>
     <p>{% include "includes/domain_application.html" %}</p>
-    <p><button type="" class="usa-button usa-button--outline withdraw_outline">
+    <p><a class="usa-button usa-button--outline withdraw_outline">
-      Withdraw Request</button>
+      Withdraw Request</a>
     </p>
   </div>
 
@@ -42,14 +42,16 @@
     {% include "includes/summary_item.html" with title='Type of organization' value=domainapplication.get_organization_type_display %}
 
     {% if domainapplication.tribe_name %}
-      {% include "includes/summary_item.html" with title='Tribal government' value=domainapplication.tribe_name%}
+      {% include "includes/summary_item.html" with title='Tribal government' value=domainapplication.tribe_name %}
+
       {% if domainapplication.federally_recognized_tribe %}
-      <p>Federally-recognized tribe</p>
+        <p>Federally-recognized tribe</p>
       {% endif %}
       
       {% if domainapplication.state_recognized_tribe %}
-      <p>State-recognized tribe</p>
+        <p>State-recognized tribe</p>
       {% endif %}
+
     {% endif %}
 
     {% if domainapplication.get_federal_type_display %}
@@ -89,7 +91,6 @@
     {% endif %}
 
     {% include "includes/summary_item.html" with title='Other employees from your organization' value=domainapplication.other_contacts.all contact='true' list='true' %}
-
     {% include "includes/summary_item.html" with title='Anything else we should know' value=domainapplication.anything_else|default:"No" %}
 
   </div>

src/registrar/templates/application_withdraw_confirmation.html

@@ -0,0 +1,21 @@
+{% extends 'base.html' %}
+
+{% block title %}Withdraw request for {{ domainapplication.requested_domain.name }}{% endblock %}
+{% load static url_helpers %}
+
+{% block content %}
+<div class="grid-container">
+<div class="grid-col desktop:grid-offset-2 desktop:grid-col-8">
+ 
+
+<h1>Withdraw request for {{ domainapplication.requested_domain.name }}?</h1>
+
+<p>If you withdraw your request we won't review it. Once you withdraw your request you'll be able to edit it or completely remove it. </p>
+
+<p><a href="{% url 'application-withdrawn' domainapplication.id %}" class="usa-button withdraw">Withdraw request</a>  
+  <a href="{% url 'application-status' domainapplication.id %}">Cancel</a></p>
+
+</div>
+
+</div>
+{% endblock %}

src/registrar/templates/home.html

@@ -88,7 +88,7 @@
           <td data-sort-value="{{ application.created_at|date:"U" }}" data-label="Date created">{{ application.created_at|date }}</td>
           <td data-label="Status">{{ application.status|title }}</td>
           <td>
-             {% if application.status == "started" %}
+             {% if application.status == "started" or application.status == "withdrawn"  %}
               <a href="{% url 'edit-application' application.pk %}">
               <svg class="usa-icon" aria-hidden="true" focusable="false" role="img" width="24">
                 <use xlink:href="{%static 'img/sprite.svg'%}#edit"></use>

src/registrar/templates/includes/ao_example.html

@@ -23,10 +23,10 @@
   {% endif %}
 
 {% elif organization_type == 'city' %}
-  <p>Domain requests from cities must be authorized by <strong>the mayor</strong> or the equivalent <strong>highest elected official.</strong></p>
+  <p>Domain requests from cities must be authorized by <strong>the mayor</strong> or the equivalent <strong>highest-elected official.</strong></p>
 
 {% elif organization_type == 'county' %}
-  <p>Domain requests from counties must be authorized by the <strong>chair of the county commission </strong>or <strong>the equivalent highest elected official.</strong></p>
+  <p>Domain requests from counties must be authorized by the <strong>chair of the county commission </strong>or <strong>the equivalent highest-elected official.</strong></p>
 
 {% elif organization_type == 'interstate' %}
   <p>Domain requests from interstate organizations must be authorized by the <strong>highest-ranking executive</strong> (president, director, chair, or equivalent) or <strong>one of the state’s governors or Chief Information Officers.</strong></p>

src/registrar/tests/test_views.py

@@ -1311,3 +1311,32 @@ class TestApplicationStatus(TestWithUser, WebTest):
         self.assertContains(detail_page, "testy@town.com")
         self.assertContains(detail_page, "Admin Tester")
         self.assertContains(detail_page, "Status:")
+
+    def test_application_withdraw(self):
+        """Checking application status page"""
+        application = self._completed_application()
+        application.save()
+
+        home_page = self.app.get("/")
+        self.assertContains(home_page, "citystatus.gov")
+        # click the "Manage" link
+        detail_page = home_page.click("Manage")
+        self.assertContains(detail_page, "citystatus.gov")
+        self.assertContains(detail_page, "Chief Tester")
+        self.assertContains(detail_page, "testy@town.com")
+        self.assertContains(detail_page, "Admin Tester")
+        self.assertContains(detail_page, "Status:")
+        # click the "Withdraw request" button
+        withdraw_page = detail_page.click("Withdraw Request")
+        self.assertContains(withdraw_page, "Withdraw request for")
+        home_page = withdraw_page.click("Withdraw request")
+        # confirm that it has redirected, and the status has been updated to withdrawn
+        self.assertRedirects(
+            home_page,
+            "/",
+            status_code=302,
+            target_status_code=200,
+            fetch_redirect_response=True,
+        )
+        home_page = self.app.get("/")
+        self.assertContains(home_page, "Withdrawn")

src/registrar/views/application.py

@@ -1,7 +1,7 @@
 import logging
 
 from django.contrib.auth.mixins import LoginRequiredMixin
-from django.http import Http404, HttpResponse
+from django.http import Http404, HttpResponse, HttpResponseRedirect
 from django.shortcuts import redirect, render
 from django.urls import resolve, reverse
 from django.utils.translation import gettext_lazy as _
@@ -14,6 +14,8 @@ from registrar.models import DomainApplication
 from registrar.utility import StrEnum
 from registrar.views.utility import StepsHelper
 
+from .utility import DomainPermission
+
 logger = logging.getLogger(__name__)
 
 
@@ -481,5 +483,24 @@ class ApplicationStatus(generic.DetailView):
     template_name = "application_status.html"
 
     def get_context_data(self, **kwargs):
+        """Get context details to process information from application"""
         context = super(ApplicationStatus, self).get_context_data(**kwargs)
         return context
+
+
+class ApplicationWithdraw(LoginRequiredMixin, generic.DetailView, DomainPermission):
+    model = DomainApplication
+    template_name = "application_withdraw_confirmation.html"
+    """ The page above will display asking user to confirm if they want to withdraw;
+
+    Note it uses "DomainPermission" from Domain to ensure that the person who
+    applied only have access to withdraw the request
+    """
+
+    def updatestatus(request, pk):
+        """If user click on withdraw confirm button, it will be updated to withdraw
+        and send back to homepage"""
+        application = DomainApplication.objects.get(id=pk)
+        application.status = "withdrawn"
+        application.save()
+        return HttpResponseRedirect(reverse("home"))

src/zap.conf

@@ -52,6 +52,8 @@
 10038	OUTOFSCOPE	http://app:8080/users
 10038	OUTOFSCOPE	http://app:8080/users/add
 10038	OUTOFSCOPE	http://app:8080/delete
+10038	OUTOFSCOPE	http://app:8080/withdraw 
+10038	OUTOFSCOPE	http://app:8080/withdrawconfirmed 
 # This URL always returns 404, so include it as well.
 10038	OUTOFSCOPE	http://app:8080/todo
 # OIDC isn't configured in the test environment and DEBUG=True so this gives a 500 without CSP headers