Use Maven repository on GCS for Cloud Build

This CL changes the Cloud Build flows to retrieve dependencies from our self-hosted GCS repository, to ensure that the release build are reproducible and hermetic (Note that it is still not truely reproducible as the dependency publishing process will override any existing artifacts in GCS with the current artifacts in Maven central. This is an issue that we should fix later).

There are a couple of changes involved to get this working:

1. Changed internal repo location to pull from the new repo.

2. Remove jcenter repo. It is only used to pull in the docker gradle plugin, which is not used. We instead build the deploy jar file with Gradle and build the docker image with a Dockerfile. The docker gradle plugin artifacts uploaded to GCS cannot be read because it is using some special classifier which seems to not be preserved when uploading. The java application plugin is also removed because it is only used by the docker gradle plugin.

3. Removed netty tcnative library classifier. It does not appear to be actually used (the jar downloaded from Maven central is an uber jar) and the classifier again interferes with downloading the artifacts from GCS.

4. Removed the cyclic dependency of the util project on itself. It was added because the nebula linter wanted it, which I think is an erroneous warning which should be reported upstream. The cyclic dependency was not a problem before (for yet unknown reasons), but it seems like when we force the dependency resolution (by calling project.generateDependencyPublications during configuration stage) it exacerbated the hidden issue and caused a cyclic task dependency in the util project, which is fatal. Now Nebula will complain again, but the warning is considered benign and will not cause the build to fail.

5. Added the nebula dependency lock files. We need these files when using the GCS maven repo because the we only upload artifacts after conflict resolution to GCS. If both v1 and v2 of the same library are requested in the dependency graph, only one will be uploaded. If we do not have the lock files in place, when building from GCS maven repo, Gradle will try to first find both v1 and v2 in the repo (which fails because v1 is not present in the repo), before proceeding to select v2 to use.

6. Refactored the code to upload Maven artifacts to GCS. We need to manually edit the POM file to reproduce the dependencies for each artifact so that they are all put in the classpath during compilation. Before, the POM files do not have any dependency information, which causes compilation to fail because transitive dependencies are not loaded (even though they are present in the GCS repo).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=233408051

cloudbuild-nomulus.yaml

@@ -19,12 +19,17 @@ steps:
   args: ['chmod', '-R', '777', '.']
 # Clone the private repo and merge its contents.
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['source', 'repos', 'clone', 'nomulus-config']
+  args: ['source', 'repos', 'clone', 'nomulus-internal']
 - name: 'alpine'
-  args: ['sh', '-c', 'cp -r nomulus-config/* .']
+  args: ['sh', '-c', 'cp -r nomulus-internal/* .']
 # Build the deployment files.
 - name: 'google/cloud-sdk'
-  args: ['./gradlew', 'stage', '-x', 'autoLintGradle']
+  args:
+    - './gradlew'
+    - 'stage'
+    - '-x'
+    - 'autoLintGradle'
+    - '-PrepositoryUrl=gcs://domain-registry-maven-repository'
   dir: 'gradle'
 # Tar the deployment files as we cannot upload directories to GCS.
 - name: 'alpine'

cloudbuild-proxy.yaml

@@ -19,12 +19,17 @@ steps:
   args: ['chmod', '-R', '777', '.']
 # Clone the private repo merge its contents.
 - name: 'gcr.io/cloud-builders/gcloud'
-  args: ['source', 'repos', 'clone', 'nomulus-config']
+  args: ['source', 'repos', 'clone', 'nomulus-internal']
 - name: 'alpine'
-  args: ['sh', '-c', 'cp -r nomulus-config/* .']
+  args: ['sh', '-c', 'cp -r nomulus-internal/* .']
 # Build the deploy jar.
 - name: 'openjdk:8-slim'
-  args: ['./gradlew', ':proxy:deployJar', '-x', 'autoLintGradle']
+  args:
+    - './gradlew'
+    - ':proxy:deployJar'
+    - '-x'
+    - 'autoLintGradle'
+    - '-PrepositoryUrl=gcs://domain-registry-maven-repository'
   dir: 'gradle'
 # Build the docker image.
 - name: 'gcr.io/cloud-builders/docker'

gradle/build.gradle

@@ -1,7 +1,16 @@
 buildscript {
+  ext.repositoryUrl = project.findProperty('repositoryUrl')
+  ext.publishUrl = project.findProperty('publishUrl')
   repositories {
-    jcenter()
+    if (repositoryUrl == null) {
-    mavenCentral()
+      println "Using Maven central..."
+      mavenCentral()
+    } else {
+      maven {
+        println "Using GCS Maven repo..."
+        url repositoryUrl
+      }
+    }
   }
 
   // Lock buildscript dependencies.
@@ -19,11 +28,10 @@ buildscript {
 
 plugins {
   id 'maven-publish'
-  id 'nebula.lint' version '10.3.5'
+  id 'nebula.lint' version '10.4.2'
   // Config helper for annotation processors such as AutoValue and Dagger.
   // Ensures that source code is generated at an appropriate location.
   id 'net.ltgt.apt' version '0.19' apply false
-  id 'com.bmuschko.docker-java-application' version '4.0.4' apply false
   id 'net.ltgt.errorprone' version '0.6.1'
   id 'checkstyle'
   id "com.moowork.node" version "1.2.0"
@@ -56,20 +64,75 @@ task stage {
   description = 'Generates application directories for all services.'
 }
 
+
+if (publishUrl != null) {
+  publishing {
+    repositories {
+      maven {
+        url publishUrl
+      }
+    }
+  }
+}
+
+ext.processedDependencies = [] as Set<String>
+ext.processDependencies = { Set<ResolvedDependency> deps ->
+
+  if (deps.isEmpty()) {
+    return
+  }
+  deps.each { ResolvedDependency dep ->
+    if (dep.moduleGroup == "nomulus" ||
+        rootProject.processedDependencies.contains(dep.module.toString())) {
+      return
+    }
+    def name = "${dep.moduleGroup}_${dep.moduleName}_${dep.moduleVersion}"
+    rootProject.publishing {
+      publications {
+        "${name}"(MavenPublication) {
+          groupId = dep.moduleGroup
+          artifactId = dep.moduleName
+          version = dep.moduleVersion
+          dep.moduleArtifacts.each { moduleArtifact ->
+            artifact(moduleArtifact.file) {
+              classifier = moduleArtifact.classifier
+            }
+          }
+          if (!dep.children.isEmpty()) {
+            pom.withXml {
+              def dependenciesNode = asNode().appendNode("dependencies")
+              dep.children.each {
+                def dependencyNode =
+                    dependenciesNode.appendNode("dependency")
+                dependencyNode.appendNode("groupId", it.moduleGroup)
+                dependencyNode.appendNode("artifactId", it.moduleName)
+                dependencyNode.appendNode("version", it.moduleVersion)
+              }
+            }
+          }
+        }
+      }
+    }
+    rootProject.processedDependencies.add(dep.module.toString())
+    rootProject.processDependencies(dep.children)
+  }
+}
+
 allprojects {
   // Skip no-op project
   if (project.name == 'services') return
 
-  publishing {
+  repositories {
-    repositories {
+    if (rootProject.repositoryUrl == null) {
+      mavenCentral()
+    } else {
       maven {
-        url = project.findProperty('repositoryUrl')
+        url rootProject.repositoryUrl
       }
     }
   }
 
-  ext.getDistinctResolvedArtifacts = {
+  ext.generateDependencyPublications = {
-    def distinctResolvedArtifacts = [:]
     def allconfigs = []
 
     allconfigs.addAll(configurations)
@@ -80,34 +143,8 @@ allprojects {
       if (!it.isCanBeResolved()) {
         return
       }
-      it.resolvedConfiguration.resolvedArtifacts.each { resolvedArtifact ->
+      rootProject.processDependencies(
-        if (resolvedArtifact.id.componentIdentifier.displayName in
+          it.resolvedConfiguration.firstLevelModuleDependencies)
-          ['project :core', 'project :proxy', 'project :util', 'project :third_party']) {
-          return
-        }
-        distinctResolvedArtifacts[resolvedArtifact.id.toString()] = resolvedArtifact
-      }
-    }
-
-    return distinctResolvedArtifacts
-  }
-
-  ext.generateDependencyPublications = {
-    def distinctResolvedArtifacts = project.ext.getDistinctResolvedArtifacts()
-
-    distinctResolvedArtifacts.values().eachWithIndex { resolvedArtifact, n ->
-      project.publishing {
-        publications {
-          "maven${n}"(MavenPublication) {
-            artifact(resolvedArtifact.file) {
-              groupId = resolvedArtifact.moduleVersion.id.group
-              artifactId = resolvedArtifact.moduleVersion.id.name
-              version = resolvedArtifact.moduleVersion.id.version
-              classifier = resolvedArtifact.classifier
-            }
-          }
-        }
-      }
     }
   }
 
@@ -127,7 +164,7 @@ allprojects {
     def artifactId = resolvedArtifact.moduleVersion.id.name
     def version = resolvedArtifact.moduleVersion.id.version
     def relativeFileName =
-      [groupId, artifactId, 'README.domainregistry'].join('/')
+        [groupId, artifactId, 'README.domainregistry'].join('/')
     def metadataFile = new File(gitRepositoryPath, relativeFileName)
     metadataFile.parentFile.mkdirs()
     def writer = metadataFile.newWriter()
@@ -137,8 +174,8 @@ allprojects {
     writer.close()
   }
 
-  // This task generates a metadata file for each resolved dependency artifact.
+// This task generates a metadata file for each resolved dependency artifact.
-  // The file contains the name, url and version for the artifact.
+// The file contains the name, url and version for the artifact.
   task generateDependencyMetadata {
     doLast {
       def distinctResolvedArtifacts = project.ext.getDistinctResolvedArtifacts()
@@ -146,16 +183,16 @@ allprojects {
 
       distinctResolvedArtifacts.values().each { resolvedArtifact ->
         def artifact = new org.sonatype.aether.util.artifact.DefaultArtifact(
-          resolvedArtifact.id.componentIdentifier.toString())
+            resolvedArtifact.id.componentIdentifier.toString())
         for (repository in project.repositories) {
           def mavenRepository = (MavenArtifactRepository) repository
           def repositoryUri = URI.create(mavenRepository.url.toString())
           def artifactUri = repositoryUri.resolve(defaultLayout.getPath(artifact))
           if (project.ext.urlExists(artifactUri.toURL())) {
             project.ext.writeMetadata(
-              resolvedArtifact,
+                resolvedArtifact,
-              artifactUri.toURL(),
+                artifactUri.toURL(),
-              project.findProperty('privateRepository') + "/${project.name}")
+                project.findProperty('privateRepository') + "/${project.name}")
             break
           }
         }
@@ -175,11 +212,6 @@ subprojects {
     }
   }
 
-  repositories {
-    jcenter()
-    mavenCentral()
-  }
-
   // Lock application dependencies.
   dependencyLocking {
     lockAllConfigurations()
@@ -276,16 +308,16 @@ subprojects {
     options.compilerArgs << "-Werror"
     options.errorprone.disableWarningsInGeneratedCode = true
     options.errorprone.errorproneArgumentProviders.add([
-            asArguments: {
+        asArguments: {
-              return ['-XepExcludedPaths:.*/build/generated/.*']
+          return ['-XepExcludedPaths:.*/build/generated/.*']
-            }] as CommandLineArgumentProvider)
+        }] as CommandLineArgumentProvider)
   }
 
   version = '1.0'
   sourceCompatibility = '1.8'
   targetCompatibility = '1.8'
 
-  compileJava {options.encoding = "UTF-8"}
+  compileJava { options.encoding = "UTF-8" }
 
   gradleLint.rules = [
       // Checks if Gradle wrapper is up-to-date
@@ -299,9 +331,9 @@ subprojects {
 
   if (project.name == 'third_party') return
 
-  // Path to code generated with annotation processors. Note that this path is
+// Path to code generated with annotation processors. Note that this path is
-  // chosen by the 'net.ltgt.apt' plugin, and may change if IDE-specific plugins
+// chosen by the 'net.ltgt.apt' plugin, and may change if IDE-specific plugins
-  // are applied, e.g., 'idea' or 'eclipse'
+// are applied, e.g., 'idea' or 'eclipse'
   def aptGeneratedDir = "${project.buildDir}/generated/source/apt/main"
   def aptGeneratedTestDir = "${project.buildDir}/generated/source/apt/test"
 
@@ -311,13 +343,13 @@ subprojects {
     main {
       java {
         srcDirs = [
-            project(':').javaDir,
+            rootProject.javaDir,
             aptGeneratedDir
         ]
       }
       resources {
         srcDirs = [
-            project(':').javaDir
+            rootProject.javaDir
         ]
         exclude commonlyExcludedResources
       }
@@ -325,13 +357,13 @@ subprojects {
     test {
       java {
         srcDirs = [
-            project(':').javatestsDir,
+            rootProject.javatestsDir,
             aptGeneratedTestDir
         ]
       }
       resources {
         srcDirs = [
-            project(':').javatestsDir,
+            rootProject.javatestsDir,
         ]
         exclude commonlyExcludedResources
       }
@@ -339,7 +371,7 @@ subprojects {
   }
 
   test {
-      testLogging.showStandardStreams = Boolean.parseBoolean(showAllOutput)
+    testLogging.showStandardStreams = Boolean.parseBoolean(showAllOutput)
   }
 
   if (project.name == 'core') return
@@ -363,3 +395,5 @@ subprojects {
     }
   }
 }
+
+generateDependencyPublications()

gradle/core/build.gradle

@@ -579,4 +579,4 @@ task nomulus(type: Jar) {
   dependsOn project(':third_party').jar
 }
 
-ext.generateDependencyPublications()
+generateDependencyPublications()

gradle/dependencies.gradle

@@ -104,7 +104,6 @@ ext {
       'jline:jline:1.0',
       'joda-time:joda-time:2.9.2',
       'junit:junit:4.12',
-      'nomulus:util:1.0',
       'org.apache.avro:avro:1.8.2',
       'org.apache.beam:beam-runners-direct-java:2.2.0',
       'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.1.0',

gradle/proxy/build.gradle

@@ -1,18 +1,3 @@
-apply plugin: 'com.google.osdetector'
-apply plugin: 'application'
-apply plugin: 'com.bmuschko.docker-java-application'
-
-// TODO(jianglai): use plugins block once the osdetctor v1.6.0 works with it.
-// see: https://github.com/google/osdetector-gradle-plugin/issues/15
-buildscript {
-  repositories {
-    mavenCentral()
-  }
-  dependencies {
-    classpath 'com.google.gradle:osdetector-gradle-plugin:1.6.0'
-  }
-}
-
 sourceSets {
   main {
     resources {
@@ -22,8 +7,6 @@ sourceSets {
   }
 }
 
-mainClassName = 'google.registry.proxy.ProxyServer'
-
 task deployJar(type: Jar) {
   manifest {
     attributes 'Main-Class': 'google.registry.proxy.ProxyServer'
@@ -74,7 +57,7 @@ dependencies {
 
   runtime deps['com.google.flogger:flogger-system-backend']
   runtime deps['com.google.auto.value:auto-value']
-  runtime deps['io.netty:netty-tcnative-boringssl-static'] + ":${osdetector.classifier}"
+  runtime deps['io.netty:netty-tcnative-boringssl-static']
 
   testCompile deps['com.google.monitoring-client:contrib']
   testCompile deps['com.google.truth:truth']
@@ -92,12 +75,4 @@ dependencies {
   testAnnotationProcessor deps['com.google.dagger:dagger-compiler']
 }
 
-docker {
+generateDependencyPublications()
-  javaApplication {
-    // TODO(jianglai): Peg to a specific hash to enable reproducible build.
-    baseImage = 'openjdk:8-jre-alpine'
-    ports = [30000, 30001, 30002, 30011, 30012]
-  }
-}
-
-ext.generateDependencyPublications()

gradle/util/build.gradle

@@ -1,6 +1,5 @@
 dependencies {
   def deps = rootProject.dependencyMap
-
   compile deps['com.google.appengine:appengine-api-1.0-sdk']
   compile deps['com.google.appengine:appengine-testing']
   compile deps['com.google.code.findbugs:jsr305']
@@ -13,7 +12,6 @@ dependencies {
   compile deps['javax.mail:mail']
   compile deps['javax.xml.bind:jaxb-api']
   compile deps['joda-time:joda-time']
-  compile deps['nomulus:util']
   compile deps['org.yaml:snakeyaml']
   testCompile deps['com.google.appengine:appengine-api-stubs']
   testCompile deps['com.google.guava:guava-testlib']
@@ -28,3 +26,5 @@ dependencies {
   annotationProcessor deps['com.google.dagger:dagger-compiler']
   testAnnotationProcessor deps['com.google.dagger:dagger-compiler']
 }
+
+generateDependencyPublications()