Setup initial CI gating on tests and add linting tests (#85)

* add flake, black, mypy, and bandit to run

* fixes issues flake and black complained about

* make mypy run successfully, add configuration files rather than specifying in ci

* respond to feedback

* configure bandit, ignore a file used only in local development

.github/workflows/test.yaml

@@ -0,0 +1,35 @@
+# test.yml
+name: Testing
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+
+jobs:
+  python-linting:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+      - name: Install linters
+        run: |
+          python -m pip install --upgrade pip
+          pip install bandit black flake8 mypy types-requests
+      - name: Lint with flake8
+        working-directory: ./src
+        run: flake8 . --count --show-source --statistics
+      - name: Check formatting with black
+        working-directory: ./src
+        run: black --check .
+      - name: Run type checking
+        working-directory: ./src
+        run: mypy .
+      - name: Run bandit security scanning
+        working-directory: ./src
+        run: bandit -r .

docs/research/scripts/icann_lookup.py

@@ -11,25 +11,29 @@ NOTE: This requries python-whois and argparse to be installed.
 
 import csv
 import requests
-import whois # this is python-whois
+import whois  # this is python-whois
 import argparse
 import sys
 from pathlib import Path
-GOV_URLS_CSV_URL = "https://raw.githubusercontent.com/GSA/govt-urls/master/1_govt_urls_full.csv"
+
+GOV_URLS_CSV_URL = (
+    "https://raw.githubusercontent.com/GSA/govt-urls/master/1_govt_urls_full.csv"
+)
 
 data = requests.get(GOV_URLS_CSV_URL).text
-csv_data = list(csv.reader(data.splitlines(), delimiter=','))
+csv_data = list(csv.reader(data.splitlines(), delimiter=","))
 domains = csv_data[1:]
-fields = csv_data[0] + ['Registrar']
+fields = csv_data[0] + ["Registrar"]
+
 
 def check_registration(name):
     try:
         domain_info = whois.whois(name)
-        return domain_info['registrar']
+        return domain_info["registrar"]
     except KeyboardInterrupt:
         sys.exit(1)
     except:
-        print(f'Something went wrong with that domain lookup for {name}, continuing...')
+        print(f"Something went wrong with that domain lookup for {name}, continuing...")
 
 
 def main(domain):
@@ -40,7 +44,11 @@ def main(domain):
     else:
         for idx, domain in enumerate(domains):
             domain_name = domain[0].lower()
-            if domain_name.endswith('.com') or domain_name.endswith('.edu') or domain_name.endswith('.net'):
+            if (
+                domain_name.endswith(".com")
+                or domain_name.endswith(".edu")
+                or domain_name.endswith(".net")
+            ):
                 print(idx)
                 print(domain_name)
                 registrar = check_registration(domain_name)
@@ -48,15 +56,17 @@ def main(domain):
 
     Path("../data").mkdir(exist_ok=True)
 
-    with open('../data/registrar_data.csv', 'w') as f:
+    with open("../data/registrar_data.csv", "w") as f:
         writer = csv.writer(f)
         writer.writerow(fields)
         writer.writerows(full_data)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     cl = argparse.ArgumentParser(description="This performs ICANN lookups on domains.")
-    cl.add_argument("--domain", help="finds the registrar for a single domain", default=None)
+    cl.add_argument(
+        "--domain", help="finds the registrar for a single domain", default=None
+    )
     args = cl.parse_args()
 
     sys.exit(main(args.domain))

docs/research/scripts/response_codes.py

@@ -10,12 +10,15 @@ This script can be run locally to generate data and currently takes some time to
 import csv
 import requests
 
-DOMAIN_LIST_URL = "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv"
+DOMAIN_LIST_URL = (
+    "https://raw.githubusercontent.com/cisagov/dotgov-data/main/current-full.csv"
+)
+
+data = requests.get(DOMAIN_LIST_URL).content.decode("utf-8")
+csv_data = list(csv.reader(data.splitlines(), delimiter=","))
+domains = csv_data[1:]
+fields = csv_data[0] + ["Response"]
 
-data = requests.get(DOMAIN_LIST_URL).content.decode('utf-8')
-csv_data = list(csv.reader(data.splitlines(), delimiter=','))
-domains =  csv_data[1:]
-fields = csv_data[0] + ['Response']
 
 def check_status_response(domain):
     try:
@@ -24,13 +27,14 @@ def check_status_response(domain):
         response = type(e).__name__
     return response
 
+
 full_data = []
 for domain in domains:
     domain_name = domain[0]
     response = check_status_response(domain_name)
     full_data.append(domain + [response])
 
-with open('../data/response_codes.csv', 'w') as f:
+with open("../data/response_codes.csv", "w") as f:
     writer = csv.writer(f)
     writer.writerow(fields)
     writer.writerows(full_data)

src/.bandit

@@ -0,0 +1,2 @@
+[bandit]
+exclude: docker_entrypoint.py

src/.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+max-complexity = 10
+extend-ignore = E203

src/.python-version

@@ -0,0 +1 @@
+src/runtime.txt

src/manage.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 """Django's command-line utility for administrative tasks."""
-import os
 import sys
 
 
@@ -17,5 +16,5 @@ def main():
     execute_from_command_line(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

src/pyproject.toml

@@ -0,0 +1,5 @@
+[tool.black]
+line-length=88
+
+[tool.mypy]
+ignore_missing_imports = true

src/registrar/config/settings.py

@@ -43,11 +43,8 @@ SECRET_KEY = secret("DJANGO_SECRET_KEY")
 DEBUG = env.bool("DJANGO_DEBUG", default=False)
 
 # TODO: configure and document security settings
-ALLOWED_HOSTS = [
+ALLOWED_HOSTS = ["getgov-unstable.app.cloud.gov", "get.gov"]
-    'getgov-unstable.app.cloud.gov',
+ALLOWED_CIDR_NETS = ["10.0.0.0/8"]  # nosec
-    'get.gov'
-]
-ALLOWED_CIDR_NETS = ['10.0.0.0/8']  # nosec
 USE_X_FORWARDED_HOST = True
 SESSION_COOKIE_SECURE = True
 SESSION_COOKIE_HTTPONLY = True
@@ -58,39 +55,39 @@ CORS_ALLOW_ALL_ORIGINS = False
 
 # TODO: are all of these needed? Need others?
 INSTALLED_APPS = [
-    'django.contrib.admin',
+    "django.contrib.admin",
-    'django.contrib.auth',
+    "django.contrib.auth",
-    'django.contrib.contenttypes',
+    "django.contrib.contenttypes",
-    'django.contrib.sessions',
+    "django.contrib.sessions",
-    'django.contrib.messages',
+    "django.contrib.messages",
-    'django.contrib.staticfiles',
+    "django.contrib.staticfiles",
 ]
 
 # TODO: document these for future maintainers
 MIDDLEWARE = [
-    'allow_cidr.middleware.AllowCIDRMiddleware',
+    "allow_cidr.middleware.AllowCIDRMiddleware",
-    'django.middleware.security.SecurityMiddleware',
+    "django.middleware.security.SecurityMiddleware",
-    'django.contrib.sessions.middleware.SessionMiddleware',
+    "django.contrib.sessions.middleware.SessionMiddleware",
-    'django.middleware.common.CommonMiddleware',
+    "django.middleware.common.CommonMiddleware",
-    'django.middleware.csrf.CsrfViewMiddleware',
+    "django.middleware.csrf.CsrfViewMiddleware",
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    'django.contrib.messages.middleware.MessageMiddleware',
+    "django.contrib.messages.middleware.MessageMiddleware",
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    'csp.middleware.CSPMiddleware',
+    "csp.middleware.CSPMiddleware",
 ]
 
 # TODO: decide on template engine and document in ADR
 TEMPLATES = [
     {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
-        'DIRS': [BASE_DIR / 'templates'],
+        "DIRS": [BASE_DIR / "templates"],
-        'APP_DIRS': True,
+        "APP_DIRS": True,
-        'OPTIONS': {
+        "OPTIONS": {
-            'context_processors': [
+            "context_processors": [
-                'django.template.context_processors.debug',
+                "django.template.context_processors.debug",
-                'django.template.context_processors.request',
+                "django.template.context_processors.request",
-                'django.contrib.auth.context_processors.auth',
+                "django.contrib.auth.context_processors.auth",
-                'django.contrib.messages.context_processors.messages',
+                "django.contrib.messages.context_processors.messages",
             ],
         },
     },
@@ -104,7 +101,7 @@ LOGGING = {
     "formatters": {
         "verbose": {
             "format": "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] "
-            "%(message)s",
+            "%(message)s",  # noqa
             "datefmt": "%d/%b/%Y %H:%M:%S",
         },
         "simple": {
@@ -141,15 +138,15 @@ LOGGING = {
 # https://docs.djangoproject.com/en/4.0/ref/settings/#databases
 
 DATABASES = {
-    'default': env.dj_db_url('DATABASE_URL'),
+    "default": env.dj_db_url("DATABASE_URL"),
 }
 
 
 # Internationalization
 # https://docs.djangoproject.com/en/4.0/topics/i18n/
 
-LANGUAGE_CODE = 'en-us'
+LANGUAGE_CODE = "en-us"
-TIME_ZONE = 'UTC'
+TIME_ZONE = "UTC"
 USE_I18N = True
 USE_TZ = True
 USE_L10N = True
@@ -157,13 +154,13 @@ USE_L10N = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/4.0/howto/static-files/
 
-STATIC_ROOT = BASE_DIR / 'static'
+STATIC_ROOT = BASE_DIR / "static"
 
-STATIC_URL = 'static/'
+STATIC_URL = "static/"
-ADMIN_URL = 'admin/'
+ADMIN_URL = "admin/"
 
-ROOT_URLCONF = 'registrar.config.urls'
+ROOT_URLCONF = "registrar.config.urls"
-WSGI_APPLICATION = 'registrar.config.wsgi.application'
+WSGI_APPLICATION = "registrar.config.wsgi.application"
 
 # TODO: FAC example for REST framework
 API_VERSION = "0"
@@ -186,13 +183,13 @@ REST_FRAMEWORK = {
 
 # TODO: FAC example for login.gov
 SIMPLE_JWT = {
-    'ALGORITHM': 'RS256',
+    "ALGORITHM": "RS256",
-    'AUDIENCE': None,
+    "AUDIENCE": None,
-    'ISSUER': 'https://idp.int.identitysandbox.gov/',
+    "ISSUER": "https://idp.int.identitysandbox.gov/",
-    'JWK_URL': 'https://idp.int.identitysandbox.gov/api/openid_connect/certs',
+    "JWK_URL": "https://idp.int.identitysandbox.gov/api/openid_connect/certs",
-    'LEEWAY': 0,
+    "LEEWAY": 0,
-    'AUTH_TOKEN_CLASSES': ('rest_framework_simplejwt.tokens.UntypedToken',),
+    "AUTH_TOKEN_CLASSES": ("rest_framework_simplejwt.tokens.UntypedToken",),
-    'USER_ID_CLAIM': 'sub',
+    "USER_ID_CLAIM": "sub",
 }
 TOKEN_AUTH = {"TOKEN_TTL": 3600}
 
@@ -200,4 +197,4 @@ TOKEN_AUTH = {"TOKEN_TTL": 3600}
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
 
-DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

src/registrar/config/urls.py

@@ -10,14 +10,11 @@ from django.urls import include, path
 
 from registrar.views import health
 
-urlpatterns = [
+urlpatterns = [path("admin/", admin.site.urls), path("health/", health.health)]
-    path("admin/", admin.site.urls),
-    path("health/", health.health)
-]
 
 if settings.DEBUG:
     import debug_toolbar
 
     urlpatterns = [
         path("__debug__/", include(debug_toolbar.urls)),
-    ] + urlpatterns
+    ] + urlpatterns

src/registrar/config/wsgi.py

@@ -7,8 +7,6 @@ For more information on this file, see
 https://docs.djangoproject.com/en/4.0/howto/deployment/wsgi/
 """
 
-import os
-
 from django.core.wsgi import get_wsgi_application
 
 application = get_wsgi_application()

src/registrar/views/health.py

@@ -1,4 +1,5 @@
 from django.http import HttpResponse
 
+
 def health(request):
     return HttpResponse("OK")