google-nomulus/gradle/build.gradle
jianglai b19563f69d Use Maven repository on GCS for Cloud Build
This CL changes the Cloud Build flows to retrieve dependencies from our self-hosted GCS repository, to ensure that the release build are reproducible and hermetic (Note that it is still not truely reproducible as the dependency publishing process will override any existing artifacts in GCS with the current artifacts in Maven central. This is an issue that we should fix later).

There are a couple of changes involved to get this working:

1. Changed internal repo location to pull from the new repo.

2. Remove jcenter repo. It is only used to pull in the docker gradle plugin, which is not used. We instead build the deploy jar file with Gradle and build the docker image with a Dockerfile. The docker gradle plugin artifacts uploaded to GCS cannot be read because it is using some special classifier which seems to not be preserved when uploading. The java application plugin is also removed because it is only used by the docker gradle plugin.

3. Removed netty tcnative library classifier. It does not appear to be actually used (the jar downloaded from Maven central is an uber jar) and the classifier again interferes with downloading the artifacts from GCS.

4. Removed the cyclic dependency of the util project on itself. It was added because the nebula linter wanted it, which I think is an erroneous warning which should be reported upstream. The cyclic dependency was not a problem before (for yet unknown reasons), but it seems like when we force the dependency resolution (by calling project.generateDependencyPublications during configuration stage) it exacerbated the hidden issue and caused a cyclic task dependency in the util project, which is fatal. Now Nebula will complain again, but the warning is considered benign and will not cause the build to fail.

5. Added the nebula dependency lock files. We need these files when using the GCS maven repo because the we only upload artifacts after conflict resolution to GCS. If both v1 and v2 of the same library are requested in the dependency graph, only one will be uploaded. If we do not have the lock files in place, when building from GCS maven repo, Gradle will try to first find both v1 and v2 in the repo (which fails because v1 is not present in the repo), before proceeding to select v2 to use.

6. Refactored the code to upload Maven artifacts to GCS. We need to manually edit the POM file to reproduce the dependencies for each artifact so that they are all put in the classpath during compilation. Before, the POM files do not have any dependency information, which causes compilation to fail because transitive dependencies are not loaded (even though they are present in the GCS repo).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=233408051
2019-02-11 11:24:12 -05:00

399 lines
11 KiB
Groovy

buildscript {
ext.repositoryUrl = project.findProperty('repositoryUrl')
ext.publishUrl = project.findProperty('publishUrl')
repositories {
if (repositoryUrl == null) {
println "Using Maven central..."
mavenCentral()
} else {
maven {
println "Using GCS Maven repo..."
url repositoryUrl
}
}
}
// Lock buildscript dependencies.
configurations.classpath {
resolutionStrategy.activateDependencyLocking()
}
dependencies {
classpath 'com.google.cloud.tools:appengine-gradle-plugin:1.3.3'
classpath "net.ltgt.gradle:gradle-errorprone-plugin:0.6.1"
classpath 'org.sonatype.aether:aether-api:1.13.1'
classpath 'org.sonatype.aether:aether-impl:1.13.1'
}
}
plugins {
id 'maven-publish'
id 'nebula.lint' version '10.4.2'
// Config helper for annotation processors such as AutoValue and Dagger.
// Ensures that source code is generated at an appropriate location.
id 'net.ltgt.apt' version '0.19' apply false
id 'net.ltgt.errorprone' version '0.6.1'
id 'checkstyle'
id "com.moowork.node" version "1.2.0"
}
apply from: 'dependencies.gradle'
// Provide defaults for all of the project properties.
// showAllOutput: boolean. If true, dump all test output during the build.
if (!project.hasProperty('showAllOutput')) {
ext.showAllOutput = 'false'
}
// Only do linting if the build is successful.
gradleLint.autoLintAfterFailure = false
// Paths to main and test sources.
ext.javaDir = "${rootDir}/../java"
ext.javatestsDir = "${rootDir}/../javatests"
// Tasks to deploy/stage all App Engine services
task deploy {
group = 'deployment'
description = 'Deploys all services to App Engine.'
}
task stage {
group = 'deployment'
description = 'Generates application directories for all services.'
}
if (publishUrl != null) {
publishing {
repositories {
maven {
url publishUrl
}
}
}
}
ext.processedDependencies = [] as Set<String>
ext.processDependencies = { Set<ResolvedDependency> deps ->
if (deps.isEmpty()) {
return
}
deps.each { ResolvedDependency dep ->
if (dep.moduleGroup == "nomulus" ||
rootProject.processedDependencies.contains(dep.module.toString())) {
return
}
def name = "${dep.moduleGroup}_${dep.moduleName}_${dep.moduleVersion}"
rootProject.publishing {
publications {
"${name}"(MavenPublication) {
groupId = dep.moduleGroup
artifactId = dep.moduleName
version = dep.moduleVersion
dep.moduleArtifacts.each { moduleArtifact ->
artifact(moduleArtifact.file) {
classifier = moduleArtifact.classifier
}
}
if (!dep.children.isEmpty()) {
pom.withXml {
def dependenciesNode = asNode().appendNode("dependencies")
dep.children.each {
def dependencyNode =
dependenciesNode.appendNode("dependency")
dependencyNode.appendNode("groupId", it.moduleGroup)
dependencyNode.appendNode("artifactId", it.moduleName)
dependencyNode.appendNode("version", it.moduleVersion)
}
}
}
}
}
}
rootProject.processedDependencies.add(dep.module.toString())
rootProject.processDependencies(dep.children)
}
}
allprojects {
// Skip no-op project
if (project.name == 'services') return
repositories {
if (rootProject.repositoryUrl == null) {
mavenCentral()
} else {
maven {
url rootProject.repositoryUrl
}
}
}
ext.generateDependencyPublications = {
def allconfigs = []
allconfigs.addAll(configurations)
// This only adds buildscript dependencies declare in this project.
allconfigs.addAll(buildscript.configurations)
allconfigs.each {
if (!it.isCanBeResolved()) {
return
}
rootProject.processDependencies(
it.resolvedConfiguration.firstLevelModuleDependencies)
}
}
ext.urlExists = { url ->
def connection = (HttpURLConnection) url.openConnection()
connection.setRequestMethod("HEAD")
connection.connect()
if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
return true
} else {
return false
}
}
ext.writeMetadata = { resolvedArtifact, url, gitRepositoryPath ->
def groupId = resolvedArtifact.moduleVersion.id.group
def artifactId = resolvedArtifact.moduleVersion.id.name
def version = resolvedArtifact.moduleVersion.id.version
def relativeFileName =
[groupId, artifactId, 'README.domainregistry'].join('/')
def metadataFile = new File(gitRepositoryPath, relativeFileName)
metadataFile.parentFile.mkdirs()
def writer = metadataFile.newWriter()
writer << "Name: ${artifactId}\n"
writer << "Url: ${url}\n"
writer << "Version: ${version}\n"
writer.close()
}
// This task generates a metadata file for each resolved dependency artifact.
// The file contains the name, url and version for the artifact.
task generateDependencyMetadata {
doLast {
def distinctResolvedArtifacts = project.ext.getDistinctResolvedArtifacts()
def defaultLayout = new org.sonatype.aether.util.layout.MavenDefaultLayout()
distinctResolvedArtifacts.values().each { resolvedArtifact ->
def artifact = new org.sonatype.aether.util.artifact.DefaultArtifact(
resolvedArtifact.id.componentIdentifier.toString())
for (repository in project.repositories) {
def mavenRepository = (MavenArtifactRepository) repository
def repositoryUri = URI.create(mavenRepository.url.toString())
def artifactUri = repositoryUri.resolve(defaultLayout.getPath(artifact))
if (project.ext.urlExists(artifactUri.toURL())) {
project.ext.writeMetadata(
resolvedArtifact,
artifactUri.toURL(),
project.findProperty('privateRepository') + "/${project.name}")
break
}
}
}
}
}
}
subprojects {
// Skip no-op project
if (project.name == 'services') return
buildscript {
// Lock buildscript dependencies.
configurations.classpath {
resolutionStrategy.activateDependencyLocking()
}
}
// Lock application dependencies.
dependencyLocking {
lockAllConfigurations()
}
apply plugin: 'maven-publish'
def services = [':services:default',
':services:backend',
':services:tools',
':services:pubapi']
// Set up all of the deployment projects.
if (services.contains(project.path)) {
apply plugin: 'war'
// Set this directory before applying the appengine plugin so that the
// plugin will recognize this as an app-engine standard app (and also
// obtains the appengine-web.xml from the correct location)
project.convention.plugins['war'].webAppDirName =
"../../../java/google/registry/env/crash/${project.name}"
apply plugin: 'com.google.cloud.tools.appengine'
// Get the web.xml file for the service.
war {
webInf {
from "../../../java/google/registry/env/common/${project.name}/WEB-INF"
}
}
if (project.path.contains("default")) {
def coreResourcesDir = "${rootDir}/core/build/resources/main"
war {
from("${coreResourcesDir}/google/registry/ui") {
include "registrar_bin*"
into("assets/js")
rename { String filename -> filename.replace("bin", "dbg")}
}
from("${coreResourcesDir}/google/registry/ui") {
include "registrar_bin*"
into("assets/js")
}
from("${coreResourcesDir}/google/registry/ui/css") {
include "registrar*"
into("assets/css")
}
from("${coreResourcesDir}/google/registry/ui/assets/images") {
include "**/*"
into("assets/images")
}
from("${coreResourcesDir}/google/registry/ui/html") {
include "*.html"
}
}
}
appengine {
deploy {
// TODO: change this to a variable.
project = 'domain-registry-crash'
}
}
dependencies {
compile project(':core')
}
rootProject.deploy.dependsOn appengineDeploy
rootProject.stage.dependsOn appengineStage
// Return early, do not apply the settings below.
return
}
apply plugin: 'java'
apply plugin: 'nebula.lint'
apply plugin: 'net.ltgt.apt'
apply plugin: 'net.ltgt.errorprone'
apply plugin: 'checkstyle'
// Checkstyle should run as part of the testing task
tasks.test.dependsOn tasks.checkstyleMain
tasks.test.dependsOn tasks.checkstyleTest
dependencies {
// compatibility with Java 8
errorproneJavac("com.google.errorprone:javac:9+181-r4173-1")
errorprone("com.google.errorprone:error_prone_core:2.3.2")
}
tasks.withType(JavaCompile).configureEach {
options.compilerArgs << "-Werror"
options.errorprone.disableWarningsInGeneratedCode = true
options.errorprone.errorproneArgumentProviders.add([
asArguments: {
return ['-XepExcludedPaths:.*/build/generated/.*']
}] as CommandLineArgumentProvider)
}
version = '1.0'
sourceCompatibility = '1.8'
targetCompatibility = '1.8'
compileJava { options.encoding = "UTF-8" }
gradleLint.rules = [
// Checks if Gradle wrapper is up-to-date
'archaic-wrapper',
// Checks for indirect dependencies with dynamic version spec. Best
// practice calls for declaring them with specific versions.
'undeclared-dependency',
'unused-dependency'
// TODO(weiminyu): enable more dependency checks
]
if (project.name == 'third_party') return
// Path to code generated with annotation processors. Note that this path is
// chosen by the 'net.ltgt.apt' plugin, and may change if IDE-specific plugins
// are applied, e.g., 'idea' or 'eclipse'
def aptGeneratedDir = "${project.buildDir}/generated/source/apt/main"
def aptGeneratedTestDir = "${project.buildDir}/generated/source/apt/test"
def commonlyExcludedResources = ['**/*.java', '**/BUILD']
sourceSets {
main {
java {
srcDirs = [
rootProject.javaDir,
aptGeneratedDir
]
}
resources {
srcDirs = [
rootProject.javaDir
]
exclude commonlyExcludedResources
}
}
test {
java {
srcDirs = [
rootProject.javatestsDir,
aptGeneratedTestDir
]
}
resources {
srcDirs = [
rootProject.javatestsDir,
]
exclude commonlyExcludedResources
}
}
}
test {
testLogging.showStandardStreams = Boolean.parseBoolean(showAllOutput)
}
if (project.name == 'core') return
ext.relativePath = "google/registry/${project.name}"
sourceSets.each {
it.java {
include "${project.relativePath}/"
}
it.resources {
include "${project.relativePath}/"
}
}
project(':core').sourceSets.each {
it.java {
exclude "${project.relativePath}/"
}
it.resources {
exclude "${project.relativePath}/"
}
}
}
generateDependencyPublications()