From 52ca9db15997e9b552306784e1c9d1c499f49d6e Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Tue, 9 Aug 2022 09:01:34 -0500 Subject: [PATCH 01/12] Update README --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5001d71b5..1133c1cb0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,22 @@ # .gov documentation +======================== Welcome to .gov! This is a repository to house work on the current .gov site(s). ## Onboarding -TBD \ No newline at end of file +TBD + +## Documentation + +### Architectural Decision Records + +Architectural Decision Records (ADR) are located in [docs/architecture/decisions](./docs/architecture/decisions). + +### Product Goals + +These are located in [docs/product](./docs/product). + +### Research Artifacts + +Including some helpful scanning tools. These are located in [docs/research](./docs/research) \ No newline at end of file From 447491275883f53a15f97c350abe8ad8e2ca2409 Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 10:43:57 -0500 Subject: [PATCH 02/12] Add Pipfile --- src/Pipfile | 8 ++++++++ src/Pipfile.lock | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 src/Pipfile create mode 100644 src/Pipfile.lock diff --git a/src/Pipfile b/src/Pipfile new file mode 100644 index 000000000..9ad9d9bd0 --- /dev/null +++ b/src/Pipfile @@ -0,0 +1,8 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +django = "*" + diff --git a/src/Pipfile.lock b/src/Pipfile.lock new file mode 100644 index 000000000..b6c72eaac --- /dev/null +++ b/src/Pipfile.lock @@ -0,0 +1,43 @@ +{ + "_meta": { + "hash": { + "sha256": "8b5635a4f7b069ae6661115b9eaa15466f7cd96794af5d131735a3638be101fb" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "asgiref": { + "hashes": [ + "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4", + "sha256:4a29362a6acebe09bf1d6640db38c1dc3d9217c68e6f9f6204d72667fc19a424" + ], + "markers": "python_version >= '3.7'", + "version": "==3.5.2" + }, + "django": { + "hashes": [ + "sha256:031ccb717782f6af83a0063a1957686e87cb4581ea61b47b3e9addf60687989a", + "sha256:032f8a6fc7cf05ccd1214e4a2e21dfcd6a23b9d575c6573cacc8c67828dbe642" + ], + "index": "pypi", + "version": "==4.1" + }, + "sqlparse": { + "hashes": [ + "sha256:0c00730c74263a94e5a9919ade150dfc3b19c574389985446148402998287dae", + "sha256:48719e356bb8b42991bdbb1e8b83223757b93789c00910a616a071910ca4a64d" + ], + "markers": "python_version >= '3.5'", + "version": "==0.4.2" + } + }, + "develop": {} +} From d810eee37c56bbcc082b6517d5994fe7b2a239d1 Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 10:57:22 -0500 Subject: [PATCH 03/12] Update README.md --- README.md | 4 ++++ src/README.md | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 src/README.md diff --git a/README.md b/README.md index 1133c1cb0..d18ce5a1c 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ Welcome to .gov! This is a repository to house work on the current .gov site(s). TBD +## Code + +Code is located in the [src](./src/) directory. + ## Documentation ### Architectural Decision Records diff --git a/src/README.md b/src/README.md new file mode 100644 index 000000000..3ca9fc487 --- /dev/null +++ b/src/README.md @@ -0,0 +1,21 @@ +# Registrar +======================== + +## Development + +If you're new to Django, see [Getting Started with Django](https://www.djangoproject.com/start/) for an introduction to the framework. + +### Local Setup + +* Install Docker +* Initialize the application: + + ```shell + docker-compose build + ``` +* Run the server: `docker-compose up` + +### Update Dependencies + +1. Check the [Pipfile](./src/Pipfile) for pinned dependencies and manually adjust the version numbers +1. Run `docker-compose up` and `docker-compose exec app pipenv update` to perform the upgrade and generate a new [Pipfile.lock](./src/Pipfile.lock) \ No newline at end of file From edf54a88a7704b301f603243750b9ff7797e2fc0 Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 10:57:50 -0500 Subject: [PATCH 04/12] Add Dockerfile --- src/Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/Dockerfile diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 000000000..83f72450b --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3 + +RUN apt-get update && apt-get install -y postgresql-client + +COPY Pipfile Pipfile +COPY Pipfile.lock Pipfile.lock + +RUN pip install pipenv +RUN pipenv install --system --dev From 640b8cdf1e198a5b9e49e1ba16f090ef5b7b1d91 Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 11:11:36 -0500 Subject: [PATCH 05/12] Add docker-compose and entrypoint --- src/app/docker_entrypoint.py | 56 ++++++++++++++++++++++++++++++++++++ src/docker-compose.yml | 35 ++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/app/docker_entrypoint.py create mode 100644 src/docker-compose.yml diff --git a/src/app/docker_entrypoint.py b/src/app/docker_entrypoint.py new file mode 100644 index 000000000..ea0af5ebe --- /dev/null +++ b/src/app/docker_entrypoint.py @@ -0,0 +1,56 @@ +#! /usr/bin/env python + +""" +This is a Docker entrypoint that configures the container to run +as the same uid of the user on the host container, rather than +the Docker default of root. Aside from following security best +practices, this makes it so that any files created by the Docker +container are also owned by the same user on the host system. +""" + +import sys +import os +import pwd +import subprocess # nosec + +HOST_UID = os.stat("/app").st_uid +HOST_USER = "james" + + +def does_username_exist(username): + try: + pwd.getpwnam(username) + return True + except KeyError: + return False + + +def does_uid_exist(uid): + try: + pwd.getpwuid(uid) + return True + except KeyError: + return False + + +if __name__ == "__main__": + if HOST_UID != os.geteuid(): + if not does_uid_exist(HOST_UID): + username = HOST_USER + while does_username_exist(username): + username += "0" + home_dir = "/home/%s" % username + subprocess.check_call( + [ # nosec + "useradd", + "-d", + home_dir, + "-m", + username, + "-u", + str(HOST_UID), + ] + ) + os.environ["HOME"] = "/home/%s" % pwd.getpwuid(HOST_UID).pw_name + os.setuid(HOST_UID) + os.execvp(sys.argv[1], sys.argv[1:]) # nosec diff --git a/src/docker-compose.yml b/src/docker-compose.yml new file mode 100644 index 000000000..b654c4d63 --- /dev/null +++ b/src/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3.0" +services: + app: + build: . + volumes: + - ./app:/app + links: + - db + working_dir: /app + entrypoint: python /app/docker_entrypoint.py + deploy: + restart_policy: + condition: on-failure + max_attempts: 5 + environment: + # Ensure stdout and stderr are sent straight to the terminal without buffering + - PYTHONUNBUFFERED=yup + # In case we'd like to know + - RUNNING_IN_DOCKER=yup + # How to connect to Postgre container + - DATABASE_URL=postgres://user:feedabee@db/app + # Run in development mode on our local + - DJANGO_SETTINGS_MODULE=app.settings.dev + stdin_open: true + tty: true + ports: + - "8000:8000" + command: "python" + + db: + image: postgres:latest + environment: + - POSTGRES_DB=app + - POSTGRES_USER=user + - POSTGRES_PASSWORD=feedabee From 421a47d72ae9604ac347a4706b9e8b7e2c3e04cb Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 11:12:08 -0500 Subject: [PATCH 06/12] Dedicate our work to the public --- LICENSE.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 000000000..a9eb4a921 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# License + +As a work of the [United States government](https://www.usa.gov/), this project is in the public domain within the United States of America. + +Additionally, we waive copyright and related rights in the work worldwide through the CC0 1.0 Universal public domain dedication. + +## CC0 1.0 Universal Summary + +This is a human-readable summary of the [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode). + +### No Copyright + +The person who associated a work with this deed has dedicated the work to the public domain by waiving all of their rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law. + +You can copy, modify, distribute, and perform the work, even for commercial purposes, all without asking permission. + +### Other Information + +In no way are the patent or trademark rights of any person affected by CC0, nor are the rights that other persons may have in the work or in how the work is used, such as publicity or privacy rights. + +Unless expressly stated otherwise, the person who associated a work with this deed makes no warranties about the work, and disclaims liability for all uses of the work, to the fullest extent permitted by applicable law. When using or citing the work, you should not imply endorsement by the author or the affirmer. From 70a415a3c9fc9bdde48a47cec545958284b239cd Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 11:20:34 -0500 Subject: [PATCH 07/12] Justify Docker in ADR --- .../decisions/0002-docker-for-development.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docs/architecture/decisions/0002-docker-for-development.md diff --git a/docs/architecture/decisions/0002-docker-for-development.md b/docs/architecture/decisions/0002-docker-for-development.md new file mode 100644 index 000000000..61f99b573 --- /dev/null +++ b/docs/architecture/decisions/0002-docker-for-development.md @@ -0,0 +1,25 @@ +# 2. Docker for development + +Date: 2022-08-09 + +## Status + +Accepted + +## Context + +The [Federal Audit Clearinghouse](https://github.com/GSA-TTS/FAC) project is a similar project to ours, from which we are taking many examples for reuse. FAC uses Docker. Docker is widely accepted at TTS currently and in the development community at large. + +Other options include using virtual machines or installing dependencies directly onto the host system. The first is not widely used at TTS; the second tends to cause problems with dependency management. + +## Decision + +To include a Dockerfile and docker-compose file. + +## Consequences + +We intend to deploy using containerized images; therefore, use of Docker on local will give the closest approximation to higher environments. + +The project may be perceived as less friendly to outsiders. + +A greater technical burden may be placed on new employees during onboarding. From 1fef88c7e9a9376305e02a6c2624e25dbd7f5e1c Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 11:21:00 -0500 Subject: [PATCH 08/12] Justify Django in ADR --- docs/architecture/decisions/0003-django.md | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 docs/architecture/decisions/0003-django.md diff --git a/docs/architecture/decisions/0003-django.md b/docs/architecture/decisions/0003-django.md new file mode 100644 index 000000000..aa2cf2d19 --- /dev/null +++ b/docs/architecture/decisions/0003-django.md @@ -0,0 +1,35 @@ +# 3. Django + +Date: 2022-08-09 + +## Status + +Accepted + +## Context + +The [Federal Audit Clearinghouse](https://github.com/GSA-TTS/FAC) project is a similar project to ours, from which we are taking many examples for reuse. FAC uses Django. + +Django is a web framework based on Python that has been in active development since 2005. + +It follows a very standard MVC architecture (made popular by Ruby on Rails), although in Django parlance, a “view” is a controller (whereas the views are “templates”). + +It comes with a robust object relational mapper for database access and includes a built-in admin interface suitable for simple use cases. + +Numerous other web frameworks exist in many languages, such as Ruby, PHP, Perl, Go, Elixir, etc. Other web frameworks exist in Python, such as Twisted and Flask. + +## Decision + +To use Django. + +## Consequences + +Django is well documented and approachable, even for people with limited programming experience. + +We will have a dependency on the health of the Django project and the Python ecosystem at large. + +Python is an interpreted language and will not have the same performance as compiled languages. Django is slower than its less featureful Python competitors (while being faster than Ruby on Rails, its nearest other-language competitor). + +The job market for Django developers is currently (8/2022) strong, with 7,000 open positions on a popular career site. + +Moving away from Django at a future point would require a complete rewrite of the codebase. From 4b19251155e1d5c5ac21511410cdc38f27fd2a1e Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 11:21:21 -0500 Subject: [PATCH 09/12] Justify python base image in ADR --- .../decisions/0004-docker-base-image.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/architecture/decisions/0004-docker-base-image.md diff --git a/docs/architecture/decisions/0004-docker-base-image.md b/docs/architecture/decisions/0004-docker-base-image.md new file mode 100644 index 000000000..61fc6805a --- /dev/null +++ b/docs/architecture/decisions/0004-docker-base-image.md @@ -0,0 +1,29 @@ +# 4. Docker base image + +Date: 2022-08-09 + +## Status + +Accepted + +## Context + +In containerized builds, it is rare to start from scratch. The project’s Dockerfile should specify a base image from which to build. + +The [python page on Docker Hub](https://hub.docker.com/_/python) has a discussion of what various image variants contain. + +Python images can be scoped to specific versions, such as 3.8.6, or to more generic versions, such as 3. There is a slim variant for smaller image size and variants scoped to specific versions of Debian. + +Alpine is a popular base image due to its small size and fast build times. However, it may not be the best choice for Python projects, due to its use of musl over glibc. Here’s a [blog post explaining this](https://pythonspeed.com/articles/alpine-docker-python/). + +## Decision + +To use python:3. + +## Consequences + +The python:3 image may or may not be secure enough for the long term needs of the project. + +Using python:3 ensures that minor version upgrades will be automatically performed each time the image is built. The consequence is that the code may break unexpectedly, if a minor version introduces an incompatibility. The risk of this is low, as the Python team strives for semantic versioning, but is not zero. + +Likewise, not pinning to the underlying Debian release ensures that the image will use the latest stable release at each build, with the same risk and benefit trade-off. From 6cd6799dffdd24e0041ff40759c3d5a372e23d76 Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Wed, 10 Aug 2022 11:21:43 -0500 Subject: [PATCH 10/12] Justify PostgreSQL in ADR --- .../architecture/decisions/0005-postgresql.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docs/architecture/decisions/0005-postgresql.md diff --git a/docs/architecture/decisions/0005-postgresql.md b/docs/architecture/decisions/0005-postgresql.md new file mode 100644 index 000000000..833652d86 --- /dev/null +++ b/docs/architecture/decisions/0005-postgresql.md @@ -0,0 +1,25 @@ +# 5. PostgreSQL + +Date: 2022-08-09 + +## Status + +Accepted + +## Context + +Our choices of which database engine to use are presently limited by our desire to use cloud.gov for deployment. + +Cloud.gov supports MySQL, PostgreSQL, and Oracle. + +Oracle may be subject to licensing fees. + +## Decision + +To use PostgreSQL. + +## Consequences + +All three options are approximately equally capable. The number of CVEs filed against each project differs, but not by any order of magnitude. + +The advanced administrative features of each database are not available in cloud.gov, so there is little reason to prefer one option over any other. From 80bebda0529bfd8a5134955b6aa1cd18379f09c0 Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Thu, 11 Aug 2022 09:01:02 -0500 Subject: [PATCH 11/12] Update Docker for Dev ADR based on feedback --- .../architecture/decisions/0002-docker-for-development.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/architecture/decisions/0002-docker-for-development.md b/docs/architecture/decisions/0002-docker-for-development.md index 61f99b573..37d3b06e8 100644 --- a/docs/architecture/decisions/0002-docker-for-development.md +++ b/docs/architecture/decisions/0002-docker-for-development.md @@ -8,7 +8,9 @@ Accepted ## Context -The [Federal Audit Clearinghouse](https://github.com/GSA-TTS/FAC) project is a similar project to ours, from which we are taking many examples for reuse. FAC uses Docker. Docker is widely accepted at TTS currently and in the development community at large. +Docker is widely accepted at TTS currently and in the development community at large. It works well with cloud.gov and is very portable to other cloud solutions. + +The [Federal Audit Clearinghouse](https://github.com/GSA-TTS/FAC) project is a similar project to ours, from which we are taking many examples for reuse. FAC uses Docker. Other options include using virtual machines or installing dependencies directly onto the host system. The first is not widely used at TTS; the second tends to cause problems with dependency management. @@ -20,6 +22,4 @@ To include a Dockerfile and docker-compose file. We intend to deploy using containerized images; therefore, use of Docker on local will give the closest approximation to higher environments. -The project may be perceived as less friendly to outsiders. - -A greater technical burden may be placed on new employees during onboarding. +A greater technical burden may be placed on new employees during onboarding, if they have not already installed Docker and learned to use it. From 5d1c5b86c526a4ae51c9202d8ca7c069005cd9a9 Mon Sep 17 00:00:00 2001 From: Seamus Johnston Date: Thu, 11 Aug 2022 09:01:28 -0500 Subject: [PATCH 12/12] Update Django ADR based on feedback --- docs/architecture/decisions/0003-django.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/architecture/decisions/0003-django.md b/docs/architecture/decisions/0003-django.md index aa2cf2d19..fb8064135 100644 --- a/docs/architecture/decisions/0003-django.md +++ b/docs/architecture/decisions/0003-django.md @@ -8,14 +8,14 @@ Accepted ## Context -The [Federal Audit Clearinghouse](https://github.com/GSA-TTS/FAC) project is a similar project to ours, from which we are taking many examples for reuse. FAC uses Django. - Django is a web framework based on Python that has been in active development since 2005. It follows a very standard MVC architecture (made popular by Ruby on Rails), although in Django parlance, a “view” is a controller (whereas the views are “templates”). It comes with a robust object relational mapper for database access and includes a built-in admin interface suitable for simple use cases. +The [Federal Audit Clearinghouse](https://github.com/GSA-TTS/FAC) project is a similar project to ours, from which we are taking many examples for reuse. FAC uses Django. + Numerous other web frameworks exist in many languages, such as Ruby, PHP, Perl, Go, Elixir, etc. Other web frameworks exist in Python, such as Twisted and Flask. ## Decision