diff --git a/gradle/gradle/dependency-locks/buildscript-classpath.lockfile b/gradle/gradle/dependency-locks/buildscript-classpath.lockfile index 4645211e1..ff60bebe5 100644 --- a/gradle/gradle/dependency-locks/buildscript-classpath.lockfile +++ b/gradle/gradle/dependency-locks/buildscript-classpath.lockfile @@ -13,8 +13,8 @@ com.jcraft:jzlib:1.1.1 com.moowork.gradle:gradle-node-plugin:1.2.0 com.moowork.node:com.moowork.node.gradle.plugin:1.2.0 com.netflix.nebula:gradle-lint-plugin:10.4.2 -com.netflix.nebula:nebula-gradle-interop:1.0.6 -com.netflix.nebula:nebula-test:7.2.3 +com.netflix.nebula:nebula-gradle-interop:1.0.7 +com.netflix.nebula:nebula-test:7.2.4 commons-codec:commons-codec:1.9 commons-io:commons-io:2.5 commons-lang:commons-lang:2.6 diff --git a/gradle/proxy/Dockerfile b/gradle/proxy/Dockerfile index bad276559..545c84e1e 100644 --- a/gradle/proxy/Dockerfile +++ b/gradle/proxy/Dockerfile @@ -1,4 +1,3 @@ -# TODO(jianglai): Peg to a specific sha256 hash to enable reproducible build. FROM gcr.io/distroless/java ADD build/libs/proxy_server.jar . ENTRYPOINT ["java", "-jar", "proxy_server.jar"] diff --git a/builder/Dockerfile b/release/builder/Dockerfile similarity index 78% rename from builder/Dockerfile rename to release/builder/Dockerfile index ba3d4e542..6da5663db 100644 --- a/builder/Dockerfile +++ b/release/builder/Dockerfile @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This Dockerfile builds an image that can be used to build the nomulus app. -# We need the following programs during Gradle build: +# This Dockerfile builds an image that can be used in Google Cloud Build. +# We need the following programs to build the Nomulus app and the proxy: # 1. Java 8 for compilation. # 2. Node.js/NPM for JavaScript compilation. # 3. Google Cloud SDK for generating the WARs. +# 4. Git to manipulate the private and the merged repos. +# 5. Docker to build and push images. FROM marketplace.gcr.io/google/ubuntu1804 ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8 ADD ./build.sh . diff --git a/builder/build.sh b/release/builder/build.sh similarity index 94% rename from builder/build.sh rename to release/builder/build.sh index cf701a6da..535e1c6a4 100755 --- a/builder/build.sh +++ b/release/builder/build.sh @@ -33,6 +33,10 @@ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ | apt-key add - apt-get update -y apt-get install google-cloud-sdk-app-engine-java -y +# Install git +apt-get install git -y +# Install docker +apt-get install docker.io -y apt-get remove apt-utils locales lsb-release -y apt-get autoclean -y apt-get autoremove -y diff --git a/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml similarity index 57% rename from cloudbuild-nomulus.yaml rename to release/cloudbuild-nomulus.yaml index 1edbb071b..b2149bb0e 100644 --- a/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -16,46 +16,51 @@ # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds steps: # Set permissions correctly. Not sure why it is necessary, but it is. -- name: 'alpine' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: /bin/bash args: ['chown', '-R', 'root:root', '.'] -- name: 'alpine' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: /bin/bash args: ['chmod', '-R', '777', '.'] -# Clone the private repo and merge its contents. -- name: 'gcr.io/cloud-builders/gcloud' - args: ['source', 'repos', 'clone', 'nomulus-internal'] -- name: 'alpine' - args: ['sh', '-c', 'cp -r nomulus-internal/* .'] # Create a directory to store the artifacts -- name: 'alpine' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: /bin/bash args: ['mkdir', 'nomulus'] +# Run tests +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + args: ['./gradlew', 'test'] + dir: 'gradle' # Build the deployment files for sandbox. -- name: 'gcr.io/${PROJECT_ID}/builder' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' args: - - './gradlew' - - 'stage' - - '-x' - - 'autoLintGradle' - - '-Penvironment=sandbox' - - '-PmavenUrl=gcs://domain-registry-maven-repository/maven' - - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins' + - './gradlew' + - 'stage' + - '-x' + - 'autoLintGradle' + - '-Penvironment=sandbox' + - '-PmavenUrl=gcs://domain-registry-maven-repository/maven' + - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins' dir: 'gradle' -- name: 'alpine' - args: ['sh', './move_artifacts.sh', 'sandbox', 'nomulus'] +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: bin/bash + args: ['./move_artifacts.sh', 'sandbox', 'nomulus'] # Build the deployment files for crash. -- name: 'gcr.io/${PROJECT_ID}/builder' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' args: - - './gradlew' - - 'stage' - - '-x' - - 'autoLintGradle' - - '-Penvironment=crash' - - '-PmavenUrl=gcs://domain-registry-maven-repository/maven' - - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins' + - './gradlew' + - 'stage' + - '-x' + - 'autoLintGradle' + - '-Penvironment=crash' + - '-PmavenUrl=gcs://domain-registry-maven-repository/maven' + - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins' dir: 'gradle' -- name: 'alpine' - args: ['sh', './move_artifacts.sh', 'crash', 'nomulus'] +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: bin/bash + args: ['./move_artifacts.sh', 'crash', 'nomulus'] # Create the uber tarball including all environments. -- name: 'alpine' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: bin/bash args: ['tar', 'cvf', '../nomulus.tar', '.'] dir: 'nomulus' # The tarball to upload to GCS. diff --git a/cloudbuild-proxy.yaml b/release/cloudbuild-proxy.yaml similarity index 50% rename from cloudbuild-proxy.yaml rename to release/cloudbuild-proxy.yaml index f40afab27..0f731c165 100644 --- a/cloudbuild-proxy.yaml +++ b/release/cloudbuild-proxy.yaml @@ -3,89 +3,87 @@ # credential helper. # See: https://cloud.google.com/cloud-build/docs/build-debug-locally # Then run: -# cloud-build-local --config=cloudbuild-proxy.yaml --dryrun=false --substitutions TAG_NAME=[TAG] . +# cloud-build-local --config=cloudbuild-proxy.yaml --dryrun=false --substitutions TAG_NAME=[TAG] .. # This will create a docker image named gcr.io/[PROJECT_ID]/proxy:[TAG] locally. # The PROJECT_ID is the current project name that gcloud uses. # # To manually trigger a build on GCB, run: -# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] . +# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] .. # # To trigger a build automatically, follow the instructions below and add a trigger: # https://cloud.google.com/cloud-build/docs/running-builds/automate-builds steps: # Set permissions correctly. Not sure why it is necessary, but it is. -- name: 'alpine' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' args: ['chown', '-R', 'root:root', '.'] -- name: 'alpine' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' args: ['chmod', '-R', '777', '.'] -# Clone the private repo merge its contents. -- name: 'gcr.io/cloud-builders/gcloud' - args: ['source', 'repos', 'clone', 'nomulus-internal'] -- name: 'alpine' - args: ['sh', '-c', 'cp -r nomulus-internal/* .'] # Build the deploy jar. -- name: 'openjdk:8-slim' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' args: - - './gradlew' - - ':proxy:deployJar' - - '-x' - - 'autoLintGradle' - - '-PmavenUrl=gcs://domain-registry-maven-repository/maven' - - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins' + - './gradlew' + - ':proxy:test' + - ':proxy:deployJar' + - '-x' + - 'autoLintGradle' + - '-PmavenUrl=gcs://domain-registry-maven-repository/maven' + - '-PpluginsUrl=gcs://domain-registry-maven-repository/plugins' dir: 'gradle' # Build the docker image. -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '--tag', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}', '.'] +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + args: ['docker', 'build', '--tag', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}', '.'] dir: 'gradle/proxy' # Move config files to the working directory. This is necessary because of Spinnaker limitations. # It will concantinate `location' and `path' in the artifact field to construct the artifact # path, even though the artifact is always uploaded to the `location', and `path' can be a regular # expression. -- name: 'alpine' - args: ['sh', '-c', 'mv java/google/registry/proxy/kubernetes/* .'] +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: /bin/bash + args: ['-c', 'mv java/google/registry/proxy/kubernetes/* .'] # Replace the tag "latest" with the git tag that triggered this build. This is due to a bug in # Spinnaker where the tag is appended to the image name when the deployment pipeline is triggered # by GCB pubsub messages. The bug is fixed in https://github.com/spinnaker/echo/pull/498 and we can # remove this step and the "latest" tag in the manifests when Spinnaker 1.13 is deployed. -- name: 'alpine' - args: ['sh', '-c', 'sed -i s/:latest/:${TAG_NAME}/ proxy-*.yaml'] -# Replace project name. -- name: 'alpine' - args: ['sh', '-c', 'sed -i s/GCP_PROJECT/${PROJECT_ID}/ proxy-*.yaml'] +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + entrypoint: /bin/bash + args: ['-c', 'sed -i s/:latest/:${TAG_NAME}/ proxy-*.yaml'] # Push the image. We can't let Cloud Build's default processing do that for us # because we need to push the image before we can sign it in the following # step. -- name: 'gcr.io/cloud-builders/docker' - args: ['push', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}'] +- name: 'gcr.io/${PROJECT_ID}/builder:latest' + args: ['docker', 'push', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}'] # Get the image hash and sign it. -- name: 'gcr.io/domain-registry-dev/builder' +- name: 'gcr.io/${PROJECT_ID}/builder:latest' entrypoint: /bin/bash args: - - -c - - > - hash=$(gcloud container images list-tags \ - gcr.io/${PROJECT_ID}/proxy \ - --format="get(digest)" --filter="tags = ${TAG_NAME}") && \ - gcloud --project=${PROJECT_ID} alpha container binauthz attestations \ - sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$hash \ - --attestor=build-attestor --attestor-project=${PROJECT_ID} \ - --keyversion-project=${PROJECT_ID} --keyversion-location=global \ - --keyversion-keyring=attestor-keys --keyversion-key=signing \ - --keyversion=1 + - -c + - > + hash=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/proxy \ + --format="get(digest)" --filter="tags = ${TAG_NAME}") && \ + gcloud --project=${PROJECT_ID} alpha container binauthz attestations \ + sign-and-create --artifact-url=gcr.io/${PROJECT_ID}/proxy@$hash \ + --attestor=build-attestor --attestor-project=${PROJECT_ID} \ + --keyversion-project=${PROJECT_ID} --keyversion-location=global \ + --keyversion-keyring=attestor-keys --keyversion-key=signing \ + --keyversion=1 +# Images to upload to GCR. Even though the image has already been uploaded, we still include it +# here so that the GCB pubsub message contains it (for Spinnaker to consume). +images: ['gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}'] # Config files to upload to GCS. artifacts: objects: location: 'gs://${PROJECT_ID}-deploy/${TAG_NAME}' # This cannot be regexs because of how Spinnaker constructs artifact paths. paths: - - 'proxy-deployment-crash.yaml' - - 'proxy-deployment-sandbox.yaml' - - 'proxy-deployment-production.yaml' - - 'proxy-deployment-crash-canary.yaml' - - 'proxy-deployment-sandbox-canary.yaml' - - 'proxy-deployment-production-canary.yaml' - - 'proxy-service.yaml' - - 'proxy-service-canary.yaml' + - 'proxy-deployment-alpha.yaml' + - 'proxy-deployment-crash.yaml' + - 'proxy-deployment-sandbox.yaml' + - 'proxy-deployment-production.yaml' + - 'proxy-deployment-crash-canary.yaml' + - 'proxy-deployment-sandbox-canary.yaml' + - 'proxy-deployment-production-canary.yaml' + - 'proxy-service.yaml' + - 'proxy-service-canary.yaml' timeout: 3600s options: machineType: 'N1_HIGHCPU_8' diff --git a/release/cloudbuild-release.yaml b/release/cloudbuild-release.yaml new file mode 100644 index 000000000..9c5a818b7 --- /dev/null +++ b/release/cloudbuild-release.yaml @@ -0,0 +1,92 @@ +# To run the build locally, install cloud-build-local first. +# You will need access to a private registry, so be sure to install the docker +# credential helper. +# See: https://cloud.google.com/cloud-build/docs/build-debug-locally +# Then run: +# cloud-build-local --config=cloudbuild-release.yaml --dryrun=false \ +# --substitutions TAG_NAME=[TAG] .. +# +# To manually trigger a build on GCB, run: +# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] .. +# +# To trigger a build automatically, follow the instructions below and add a trigger: +# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds +# +# This pipeline prepares a release. The pipeline should be run against the Nomulus public repo on +# GitHub. It builds the builder and base images, and hard codes the sha256 hashes of the resulting +# images in the merged code base (internal + public) , which is tagged and pushed into the release +# repo. Actual release artifacts are built from the release repo, ensuring reproducibility. +steps: +# Check the out internal repo. +- name: 'gcr.io/cloud-builders/gcloud' + args: ['source', 'repos', 'clone', 'nomulus-internal'] +# Tag and push the internal repo. +- name: 'gcr.io/cloud-builders/git' + entrypoint: /bin/bash + args: + - -c + - | + git tag ${TAG_NAME} && git push origin ${TAG_NAME} + dir: 'nomulus-internal' +# Merge the repos. +- name: 'gcr.io/cloud-builders/git' + entrypoint: /bin/bash + args: + - -c + - | + shopt -s dotglob + rm -rf .git && rm -rf nomulus-internal/.git + cp -rf nomulus-internal/* . + rm -rf nomulus-internal +# Build the builder image and tag the proxy base image, to be uploaded later. +- name: 'gcr.io/cloud-builders/docker' + entrypoint: /bin/bash + args: + - -c + - | + docker build -t gcr.io/${PROJECT_ID}/builder:${TAG_NAME} . + docker tag gcr.io/${PROJECT_ID}/builder:${TAG_NAME} gcr.io/${PROJECT_ID}/builder:latest + docker pull gcr.io/distroless/java + docker tag gcr.io/distroless/java gcr.io/${PROJECT_ID}/base:${TAG_NAME} + docker tag gcr.io/distroless/java gcr.io/${PROJECT_ID}/base:latest + docker push gcr.io/${PROJECT_ID}/base:latest + docker push gcr.io/${PROJECT_ID}/base:${TAG_NAME} + docker push gcr.io/${PROJECT_ID}/builder:latest + docker push gcr.io/${PROJECT_ID}/builder:${TAG_NAME} + dir: 'release/builder/' +# Do text replacement in the merged repo, hardcoding image hashes. +- name: 'gcr.io/cloud-builders/gcloud' + entrypoint: /bin/bash + args: + - -c + - | + builder_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/builder \ + --format='get(digest)' --filter='tags = ${TAG_NAME}') + base_digest=$(gcloud container images list-tags gcr.io/${PROJECT_ID}/base \ + --format='get(digest)' --filter='tags = ${TAG_NAME}') + sed -i s%distroless/java%${PROJECT_ID}/base@$base_digest% gradle/proxy/Dockerfile + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-proxy.yaml + sed -i s/builder:latest/builder@$builder_digest/g release/cloudbuild-nomulus.yaml + sed -i s/GCP_PROJECT/${PROJECT_ID}/ java/google/registry/proxy/kubernetes/proxy-*.yaml +# Check out the release repo. +- name: 'gcr.io/cloud-builders/gcloud' + args: ['source', 'repos', 'clone', 'nomulus-release'] +# Tag and check in the release repo. +- name: 'gcr.io/cloud-builders/git' + entrypoint: /bin/bash + args: + - -c + - | + cp -rf nomulus-release/.git . + rm -rf nomulus-release + git config --global user.name "Cloud Build" + git config --global user.email \ + $(gcloud auth list --format='get(account)' --filter=active) + git add . + git commit -m "Release commit for tag ${TAG_NAME}" + git push origin master + git tag ${TAG_NAME} + git push origin ${TAG_NAME} +timeout: 3600s +options: + machineType: 'N1_HIGHCPU_8' diff --git a/move_artifacts.sh b/release/move_artifacts.sh similarity index 100% rename from move_artifacts.sh rename to release/move_artifacts.sh