google-nomulus/core/build.gradle
gbrodman 6e73561341 Update GCL dependency to avoid security alert (#1139)
* Update GCL dependency to avoid security alert

This required a few changes in addition to the dependency update.

- a few transitive / required dependency updates as well
- updating soyutils_usegoog.js and adding checks.js because they're
necessary as part of the Soy compilation process
- Using a trustedResourceUri in the buildSrc Soy compilation instead of
a string
- changing the arguments to the Soy-to-Java compiler to comply with the
new version
- Moving all Soy UI files to be in the registrar directory. This was
not the case before due to previous thinking that we'd have separate
admin and registrar consoles -- this is no longer the case so it's no
longer necessary. This necessitated various refactorings and reference
changes.
  - The new soy-to-javascript compiler requires this, as it removes the
  "deps" param that we were previously using to say "use the general UI
  utils as dependencies for the registrar-console files".
- Creating a SQL environment and loading test data in the test server
main method -- previously, the local test server did not work.
- Fix some JS code that was referencing now-deleted library functions
- Removal of the Karma tests, as the karma-closure library hasn't been
updated since 2018 and it no longer works. We never noticed any errors
from the Karma tests, we never change the JS, and we have the
Java+Selenium screenshot differ tests to test the UI anyway.
2021-05-17 13:21:26 -04:00

1083 lines
40 KiB
Groovy

// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import com.google.common.base.CaseFormat
import java.util.Optional
plugins {
id 'java-library'
id 'maven-publish'
}
// Path to code generated by ad hoc tasks in this project. A separate path is
// used for easy inspection.
def generatedDir = "${project.buildDir}/generated/sources/custom/java/main"
def resourcesDir = "${project.buildDir}/resources/main"
def screenshotsDir = "${project.buildDir}/screenshots"
def screenshotsForGoldensDir = "${project.buildDir}/screenshots_for_goldens"
def newGoldensDir = "${project.buildDir}/new_golden_images"
def goldensDir =
"${javaTestDir}/google/registry/webdriver/goldens/chrome-linux"
def jsDir = "${project.projectDir}/src/main/javascript"
// Tests that conflict with (mostly unidentified) members of the main test
// suite. It is unclear if they are offenders (i.e., those that pollute global
// state) or victims.
// TODO(weiminyu): identify cause and fix offending tests.
def outcastTestPatterns = [
// Problem seems to lie with AppEngine TaskQueue for test.
"google/registry/batch/DeleteContactsAndHostsActionTest.*",
"google/registry/batch/RefreshDnsOnHostRenameActionTest.*",
"google/registry/flows/CheckApiActionTest.*",
"google/registry/flows/EppLifecycleHostTest.*",
"google/registry/flows/domain/DomainCreateFlowTest.*",
"google/registry/flows/domain/DomainUpdateFlowTest.*",
"google/registry/tools/CreateDomainCommandTest.*",
"google/registry/tools/server/CreatePremiumListActionTest.*",
]
// Tests that fail when running Gradle in a docker container, e. g. when
// building the release artifacts in Google Cloud Build.
def dockerIncompatibleTestPatterns = [
// The webdriver tests start headless Chrome in a Docker container,
// resulting in Docker-in-Docker complications. Likewise,
// GenerateSqlSchemaCommandTest and DumpGoldenSchemaCommandTest launch
// postgresql in a docker container.
"google/registry/webdriver/*",
"google/registry/tools/GenerateSqlSchemaCommandTest.*",
"google/registry/tools/DumpGoldenSchemaCommandTest.*",
// PathParameterTest includes tests which validate that file permissions are
// respected. However when running in Docker the user is root by default, so
// every file is read/write-able. There is no way to exclude specific test
// methods, so we exclude the whole test class.
"google/registry/tools/params/PathParameterTest.*",
"google/registry/persistence/PersistenceModuleTest.*",
]
// Tests that conflict with members of both the main test suite and the
// outcast suite. They seem to be affected by global states outside of
// Nomulus classes, e.g., threads and objects retained by frameworks.
// TODO(weiminyu): identify cause and fix offending tests.
def fragileTestPatterns = [
// Problem seems to lie with AppEngine TaskQueue for test.
"google/registry/cron/TldFanoutActionTest.*",
// Test Datastore inexplicably aborts transaction.
"google/registry/model/tmch/ClaimsListShardTest.*",
// Creates large object (64MBytes), occasionally throws OOM error.
"google/registry/model/server/KmsSecretRevisionTest.*",
// Changes cache timeouts and for some reason appears to have contention
// with other tests.
"google/registry/whois/WhoisCommandFactoryTest.*",
] + dockerIncompatibleTestPatterns
sourceSets {
main {
java {
srcDirs += generatedDir
}
resources {
exclude '**/*.xjb'
}
}
nonprod {
java {
compileClasspath += main.output
// Add the DB runtime classpath to nonprod so we can load the flyway
// scripts.
runtimeClasspath += main.output +
rootProject.project(":db").sourceSets.main.runtimeClasspath
}
}
test {
java {
compileClasspath += nonprod.output
runtimeClasspath += nonprod.output
// Javadoc API is deprecated in Java 11 and removed in Java 12.
// TODO(jianglai): re-enable after migrating to the new Javadoc API
if ((JavaVersion.current().majorVersion as Integer) >= 11) {
exclude 'google/registry/documentation/**'
}
}
resources {
exclude '**/*.xjb', '**/*.xsd'
}
}
}
processTestResources {
exclude '**/webdriver/*'
}
configurations {
css
jaxb
soy
closureCompiler
devtool
nonprodCompile.extendsFrom compile
nonprodRuntime.extendsFrom runtime
testCompile.extendsFrom nonprodCompile
testRuntime.extendsFrom nonprodRuntime
// Published jars that are used for server/schema compatibility tests.
// See <a href="../integration/README.md">the integration project</a>
// for details.
nomulus_test
// Exclude non-canonical servlet-api jars. Our AppEngine deployment uses
// javax.servlet:servlet-api:2.5
// For reasons we do not understand, marking the following dependencies as
// compileOnly instead of compile does not exclude them from runtimeClasspath.
all {
// servlet-api:3.1 pulled in but not used by soy compiler
exclude group: 'javax.servlet', module: 'javax.servlet-api'
// Jetty's servlet-api:2.5 implementation, pulled in by other Jetty jars
exclude group: 'org.mortbay.jetty', module: 'servlet-api'
}
}
// Known issues:
// - The (test/)compile/runtime labels are deprecated. We continue using these
// labels due to nebula-lint.
// TODO(weiminyu): switch to api/implementation labels.
// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/130 for
// issue status.
// - Nebula-lint's conflict between unused and undeclared dependency check.
// If an undeclared dependency is added, the unused-dependency check will flag
// it. For now we wrap affected dependency in gradleLint.ignore block.
// TODO(weiminyu): drop gradleLint.ignore block when issue is fixed.
// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/181 for
// issue status.
dependencies {
def deps = rootProject.dependencyMap
// Custom-built objectify jar at commit ecd5165, included in Nomulus
// release.
compile files(
"${rootDir}/third_party/objectify/v4_1/objectify-4.1.3.jar")
testRuntime files(sourceSets.test.resources.srcDirs)
compile deps['com.beust:jcommander']
compile deps['com.google.api:gax']
compile deps['com.google.api.grpc:proto-google-cloud-datastore-v1']
compile deps['com.google.api.grpc:proto-google-common-protos']
compile deps['com.google.api.grpc:proto-google-cloud-secretmanager-v1']
compile deps['com.google.api-client:google-api-client']
compile deps['com.google.api-client:google-api-client-appengine']
compile deps['com.google.api-client:google-api-client-servlet']
compile deps['com.google.monitoring-client:metrics']
compile deps['com.google.monitoring-client:stackdriver']
compile deps['com.google.api-client:google-api-client-java6']
compile deps['com.google.apis:google-api-services-admin-directory']
compile deps['com.google.apis:google-api-services-appengine']
compile deps['com.google.apis:google-api-services-bigquery']
compile deps['com.google.apis:google-api-services-cloudkms']
compile deps['com.google.apis:google-api-services-dataflow']
compile deps['com.google.apis:google-api-services-dns']
compile deps['com.google.apis:google-api-services-drive']
compile deps['com.google.apis:google-api-services-groupssettings']
compile deps['com.google.apis:google-api-services-monitoring']
compile deps['com.google.apis:google-api-services-sheets']
testCompile deps['com.google.appengine:appengine-api-stubs']
compile deps['com.google.appengine.tools:appengine-gcs-client']
compile deps['com.google.appengine.tools:appengine-mapreduce']
compile deps['com.google.appengine.tools:appengine-pipeline']
compile deps['com.google.appengine:appengine-remote-api']
compile deps['com.google.auth:google-auth-library-credentials']
compile deps['com.google.auth:google-auth-library-oauth2-http']
compile deps['com.google.cloud.bigdataoss:util']
compile deps['com.google.cloud.datastore:datastore-v1-proto-client']
compile deps['com.google.cloud.sql:jdbc-socket-factory-core']
runtimeOnly deps['com.google.cloud.sql:postgres-socket-factory']
compile deps['com.google.cloud:google-cloud-secretmanager']
compile deps['com.google.code.gson:gson']
compile deps['com.google.auto.service:auto-service-annotations']
compile deps['com.google.auto.value:auto-value-annotations']
compile deps['com.google.code.findbugs:jsr305']
compile deps['com.google.dagger:dagger']
compile deps['com.google.errorprone:error_prone_annotations']
compile deps['com.google.flogger:flogger']
runtime deps['com.google.flogger:flogger-system-backend']
compile deps['com.google.guava:guava']
gradleLint.ignore('unused-dependency') {
compile deps['com.google.gwt:gwt-user']
}
compile deps['com.google.http-client:google-http-client']
compile deps['com.google.http-client:google-http-client-appengine']
compile deps['com.google.http-client:google-http-client-jackson2']
compile deps['com.google.oauth-client:google-oauth-client']
compile deps['com.google.oauth-client:google-oauth-client-java6']
compile deps['com.google.oauth-client:google-oauth-client-jetty']
compile deps['com.google.oauth-client:google-oauth-client-appengine']
compile deps['com.google.oauth-client:google-oauth-client-servlet']
compile deps['com.google.protobuf:protobuf-java']
compile deps['com.google.re2j:re2j']
compile deps['com.google.template:soy']
compile deps['com.googlecode.json-simple:json-simple']
compile deps['com.jcraft:jsch']
testCompile deps['com.thoughtworks.qdox:qdox']
compile deps['com.zaxxer:HikariCP']
compile deps['dnsjava:dnsjava']
runtime deps['guru.nidi:graphviz-java-all-j2v8']
testCompile deps['io.github.classgraph:classgraph']
testRuntime deps['io.github.java-diff-utils:java-diff-utils']
testCompile deps['javax.annotation:javax.annotation-api']
testCompile deps['javax.annotation:jsr250-api']
compile deps['javax.mail:mail']
compile deps['javax.inject:javax.inject']
compile deps['javax.persistence:javax.persistence-api']
compile deps['javax.servlet:servlet-api']
compile deps['javax.xml.bind:jaxb-api']
compile deps['jline:jline']
compile deps['joda-time:joda-time']
compile deps['org.apache.avro:avro']
testCompile deps['org.apache.beam:beam-runners-core-construction-java']
testCompile deps['org.apache.beam:beam-runners-direct-java']
compile deps['org.apache.beam:beam-runners-google-cloud-dataflow-java']
compile deps['org.apache.beam:beam-sdks-java-core']
compile deps['org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core']
compile deps['org.apache.beam:beam-sdks-java-io-google-cloud-platform']
testCompile deps['org.apache.commons:commons-lang3']
testCompile deps['org.apache.commons:commons-text']
testCompile deps['org.apache.ftpserver:ftplet-api']
testCompile deps['org.apache.ftpserver:ftpserver-core']
compile deps['org.apache.httpcomponents:httpclient']
compile deps['org.apache.httpcomponents:httpcore']
runtime deps['org.apache.logging.log4j:log4j-core']
testCompile deps['org.apache.sshd:sshd-core']
testCompile deps['org.apache.sshd:sshd-scp']
testCompile deps['org.apache.sshd:sshd-sftp']
testCompile deps['org.apache.tomcat:tomcat-annotations-api']
compile deps['org.bouncycastle:bcpg-jdk15on']
testCompile deps['org.bouncycastle:bcpkix-jdk15on']
compile deps['org.bouncycastle:bcprov-jdk15on']
testCompile deps['com.fasterxml.jackson.core:jackson-databind']
runtime deps['org.glassfish.jaxb:jaxb-runtime']
compile deps['org.hibernate:hibernate-core']
compile deps['org.hibernate:hibernate-hikaricp']
compile deps['org.joda:joda-money']
compile deps['org.json:json']
compile deps['org.jsoup:jsoup']
testCompile deps['org.mortbay.jetty:jetty']
compile deps['org.postgresql:postgresql']
testCompile deps['org.seleniumhq.selenium:selenium-api']
testCompile deps['org.seleniumhq.selenium:selenium-chrome-driver']
testCompile deps['org.seleniumhq.selenium:selenium-java']
testCompile deps['org.seleniumhq.selenium:selenium-remote-driver']
runtimeOnly deps['org.slf4j:slf4j-jdk14']
testCompile deps['org.testcontainers:jdbc']
testCompile deps['org.testcontainers:junit-jupiter']
compile deps['org.testcontainers:postgresql']
testCompile deps['org.testcontainers:selenium']
testCompile deps['org.testcontainers:testcontainers']
compile deps['us.fatehi:schemacrawler']
compile deps['us.fatehi:schemacrawler-api']
compile deps['us.fatehi:schemacrawler-diagram']
compile deps['us.fatehi:schemacrawler-tools']
compile deps['xerces:xmlParserAPIs']
compile deps['xpp3:xpp3']
// This dependency must come after javax.mail:mail as it would otherwise
// shadow classes in package javax.mail with its own implementation.
compile deps['com.google.appengine:appengine-api-1.0-sdk']
// Known issue: nebula-lint misses inherited dependency.
compile project(':common')
testCompile project(path: ':common', configuration: 'testing')
compile project(':util')
// Import NomulusPostreSql from ':db' for compile but exclude dependencies.
compile project(path: ':db', configuration: 'compileApi')
testRuntime project(':db')
annotationProcessor deps['com.google.auto.service:auto-service']
annotationProcessor deps['com.google.auto.value:auto-value']
testAnnotationProcessor deps['com.google.auto.value:auto-value']
annotationProcessor deps['com.google.dagger:dagger-compiler']
testAnnotationProcessor deps['com.google.dagger:dagger-compiler']
annotationProcessor project(':processor')
testAnnotationProcessor project(':processor')
testCompile deps['com.google.appengine:appengine-testing']
testCompile deps['com.google.guava:guava-testlib']
testCompile deps['com.google.monitoring-client:contrib']
testCompile deps['com.google.truth:truth']
testCompile deps['com.google.truth.extensions:truth-java8-extension']
testCompile deps['org.hamcrest:hamcrest']
testCompile deps['org.hamcrest:hamcrest-core']
testCompile deps['org.hamcrest:hamcrest-library']
testCompile deps['junit:junit']
testCompile deps['org.junit.jupiter:junit-jupiter-api']
testCompile deps['org.junit.jupiter:junit-jupiter-engine']
testCompile deps['org.junit.jupiter:junit-jupiter-migrationsupport']
testCompile deps['org.junit.jupiter:junit-jupiter-params']
testCompile deps['org.junit-pioneer:junit-pioneer']
testCompile deps['org.junit.platform:junit-platform-runner']
testCompile deps['org.junit.platform:junit-platform-suite-api']
testCompile deps['org.mockito:mockito-core']
testCompile deps['org.mockito:mockito-junit-jupiter']
testCompile 'org.checkerframework:checker-qual:3.9.1'
runtime deps['org.postgresql:postgresql']
// Indirect dependency found by undeclared-dependency check. Such
// dependencies should go after all other compile and testCompile
// dependencies to avoid overriding them accidentally.
compile deps['com.google.oauth-client:google-oauth-client-java6']
// Dependencies needed for jaxb compilation.
// Use jaxb 2.2.11 because 2.3 is known to break the Ant task we use.
// TODO: upgrade jaxb versions to 2.4.0, already in beta by Sept 2018
jaxb deps['javax.xml.bind:jaxb-api']
jaxb deps['com.sun.activation:javax.activation']
jaxb deps['com.sun.xml.bind:jaxb-xjc']
jaxb deps['com.sun.xml.bind:jaxb-impl']
jaxb deps['com.sun.xml.bind:jaxb-osgi']
// Dependency needed for soy to java compilation.
soy deps['com.google.inject:guice']
soy deps['com.google.protobuf:protobuf-java']
soy deps['com.google.template:soy']
// Dependencies needed for compiling stylesheets to javascript
css deps['com.google.closure-stylesheets:closure-stylesheets']
css deps['args4j:args4j']
// Tool dependencies. used for doc generation.
compile files("${System.properties['java.home']}/../lib/tools.jar")
// Flyway classes needed to generate the golden file.
compile deps['org.flywaydb:flyway-core']
closureCompiler deps['com.google.javascript:closure-compiler']
}
task jaxbToJava {
def xsdFilesDir = "${javaDir}/google/registry/xml/xsd"
def bindingsFile = "${javaDir}/google/registry/xjc/bindings.xjb"
def pkgInfoTemplate = "${javaDir}/google/registry/xjc/package-info.java.in"
def pkgInfoMap = "${javaDir}/google/registry/xjc/package-info.map"
def outputDir = "${generatedDir}/google/registry/xjc"
inputs.dir xsdFilesDir
inputs.files bindingsFile, pkgInfoTemplate, pkgInfoMap
outputs.dir outputDir
doLast {
file(generatedDir).mkdirs()
// Temp dir to hold schema and bindings files. Files must be in the same
// directory because the bindings (.xjb) file does not declare relative
// paths to schema (.xsd) files.
def xjcTempSourceDir = file("${temporaryDir}/xjc")
xjcTempSourceDir.mkdirs()
ant.copy(
todir: "${xjcTempSourceDir}",
overwrite: true) {
fileSet(
dir: xsdFilesDir,
includes: '**.xsd')
}
ant.copy(
todir: "${xjcTempSourceDir}",
overwrite: true,
file: bindingsFile)
ant.taskdef(
name: 'xjc',
classname: 'com.sun.tools.xjc.XJCTask',
classpath: configurations.jaxb.asPath)
ant.xjc(
destdir: "${generatedDir}",
binding: "${xjcTempSourceDir}/bindings.xjb",
removeOldOutput: 'yes', extension: 'true') {
project.fileTree(
dir: new File("$xjcTempSourceDir"),
include: ['**/*.xsd'])
.addToAntBuilder(ant, 'schema', FileCollection.AntType.FileSet)
// -npa: do not generate package-info.java files. They will be generated
// below.
arg(line: '-npa -quiet -extension')
}
new File(pkgInfoMap).eachLine { line ->
def (packageName, namespace) = line.split()
ant.copy(
tofile: "${outputDir}/${packageName}/package-info.java",
overwrite: true,
file: "${pkgInfoTemplate}") {
filterSet() {
filter(token: 'PACKAGE', value: packageName)
filter(token: 'NAMESPACE', value: namespace)
}
}
}
execInBash(
"find . -name *.java -exec sed -i -e '/" + /\* <p>$/ + "/d' {} +",
generatedDir)
}
}
task soyToJava {
// Relative paths of soy directories.
def spec11SoyDir = "google/registry/reporting/spec11/soy"
def toolsSoyDir = "google/registry/tools/soy"
def registrarSoyDir = "google/registry/ui/soy/registrar"
def soyRelativeDirs = [spec11SoyDir, toolsSoyDir, registrarSoyDir]
soyRelativeDirs.each {
inputs.dir "${resourcesSourceDir}/${it}"
outputs.dir "${generatedDir}/${it}"
}
ext.soyToJava = { javaPackage, outputDirectory, soyFiles ->
javaexec {
main = "com.google.template.soy.SoyParseInfoGenerator"
classpath configurations.soy
args "--javaPackage", "${javaPackage}",
"--outputDirectory", "${outputDirectory}",
"--javaClassNameSource", "filename",
"--allowExternalCalls", "true",
"--srcs", "${soyFiles.join(',')}",
"--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
}
}
doLast {
soyToJava('google.registry.tools.soy',
"${generatedDir}/${toolsSoyDir}",
fileTree(
dir: "${resourcesSourceDir}/${toolsSoyDir}",
include: ['**/*.soy']))
soyToJava('google.registry.ui.soy.registrar',
"${generatedDir}/${registrarSoyDir}",
fileTree(
dir: "${resourcesSourceDir}/${registrarSoyDir}",
include: ['**/*.soy']))
soyToJava('google.registry.reporting.spec11.soy',
"${generatedDir}/${spec11SoyDir}",
fileTree(
dir: "${resourcesSourceDir}/${spec11SoyDir}",
include: ['**/*.soy']))
}
}
task soyToJS(type: JavaExec) {
def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy/registrar"
def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy/registrar"
inputs.dir rootSoyDirectory
outputs.dir outputSoyDirectory
def inputSoyFiles = files {
file("${rootSoyDirectory}").listFiles()
}.filter {
it.name.endsWith(".soy")
}
classpath configurations.soy
main = "com.google.template.soy.SoyToJsSrcCompiler"
args "--outputPathFormat", "${outputSoyDirectory}/{INPUT_FILE_NAME}.js",
"--allowExternalCalls", "false",
"--srcs", "${inputSoyFiles.join(',')}",
"--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt"
}
task stylesheetsToJavascript {
def cssSourceDir = "${jsDir}/google/registry/ui/css"
def outputDir = "${resourcesDir}/google/registry/ui/css"
inputs.dir cssSourceDir
outputs.dir outputDir
ext.cssCompile = { outputName, debug, cssFiles ->
javaexec {
main = "com.google.common.css.compiler.commandline.ClosureCommandLineCompiler"
classpath configurations.css
def argsBuffer = [
"--output-file", "${outputName}.css",
"--output-source-map", "${outputName}.css.map",
"--input-orientation", "LTR",
"--output-orientation", "NOCHANGE",
"--output-renaming-map", "${outputName}.css.js",
"--output-renaming-map-format", "CLOSURE_COMPILED_SPLIT_HYPHENS"
]
if (debug) {
argsBuffer.addAll(["--rename", "DEBUG", "--pretty-print"])
} else {
argsBuffer.addAll(["--rename", "CLOSURE"])
}
argsBuffer.addAll(cssFiles)
args argsBuffer
}
}
doLast {
file("${outputDir}").mkdirs()
def ignoredFiles = ["demo_css.css", "registrar_imports_raw.css"]
def sourceFiles = []
// include all CSS files that we find except for the ones explicitly ignored
fileTree(cssSourceDir).each {
if (it.name.endsWith(".css") && !ignoredFiles.contains(it.name)) {
sourceFiles << (cssSourceDir + "/" + it.name)
}
}
// The css files have to be passed to the compiler in alphabetic order to
// avoid some flaky style issues
sourceFiles.sort()
cssCompile("${outputDir}/registrar_bin", false, sourceFiles)
cssCompile("${outputDir}/registrar_dbg", true, sourceFiles)
}
}
task compileProdJS(type: JavaExec) {
def outputDir = "${resourcesDir}/google/registry/ui"
def nodeModulesDir = "${rootDir}/node_modules"
def cssSourceDir = "${resourcesDir}/google/registry/ui/css"
def jsSourceDir = "${jsDir}/google/registry/ui/js"
def externsDir = "${jsDir}/google/registry/ui/externs"
def soySourceDir = "${generatedDir}/google/registry/ui/soy"
[nodeModulesDir, cssSourceDir, jsSourceDir, externsDir, soySourceDir].each {
inputs.dir "${it}"
}
outputs.dir outputDir
classpath configurations.closureCompiler
main = 'com.google.javascript.jscomp.CommandLineRunner'
def closureArgs = []
closureArgs << "--js_output_file=${outputDir}/registrar_bin.js"
// sourcemap-related configuration
closureArgs << "--create_source_map=${outputDir}/registrar_bin.js.map"
closureArgs << "--source_map_include_content=true"
closureArgs << "--source_map_location_mapping=${rootDir}/|"
closureArgs << "--output_wrapper=\"%output% //# sourceMappingURL=registrar_bin.js.map\""
// compilation options
closureArgs << "--compilation_level=ADVANCED"
closureArgs << "--entry_point=goog:registry.registrar.main"
closureArgs << "--generate_exports"
// manually include all the required js files
closureArgs << "--js=${nodeModulesDir}/google-closure-library/**/*.js"
closureArgs << "--js=${jsDir}/*.js"
closureArgs << "--js=${cssSourceDir}/registrar_bin.css.js"
closureArgs << "--js=${jsSourceDir}/**.js"
closureArgs << "--js=${externsDir}/json.js"
closureArgs << "--js=${soySourceDir}/**.js"
args closureArgs
}
compileJava.dependsOn jaxbToJava
compileJava.dependsOn soyToJava
// The Closure JS compiler does not support Windows. It is fine to disable it if
// all we want to do is to complile the Java code on Windows.
if (!System.properties['os.name'].toLowerCase().contains('windows')) {
compileJava.dependsOn compileProdJS
assemble.dependsOn compileProdJS
}
// stylesheetsToJavascript must happen after processResources, which wipes the
// resources folder before copying data into it.
stylesheetsToJavascript.dependsOn processResources
classes.dependsOn stylesheetsToJavascript
compileProdJS.dependsOn stylesheetsToJavascript
compileProdJS.dependsOn rootProject.npmInstall
compileProdJS.dependsOn processResources
compileProdJS.dependsOn processTestResources
compileProdJS.dependsOn soyToJS
// Make testing artifacts available to be depended up on by other projects.
// TODO: factor out google.registry.testing to be a separate project.
task testJar(type: Jar) {
classifier = 'test'
from sourceSets.test.output
}
artifacts {
testRuntime testJar
}
task findGoldenImages(type: JavaExec) {
classpath = sourceSets.test.runtimeClasspath
main = 'google.registry.webdriver.GoldenImageFinder'
def arguments = []
arguments << "--screenshots_for_goldens_dir=${screenshotsForGoldensDir}"
arguments << "--new_goldens_dir=${newGoldensDir}"
arguments << "--existing_goldens_dir=${goldensDir}"
if (rootProject.findProperty("overrideExistingGoldens") == "true") {
arguments << "--override_existing_goldens=true"
}
args arguments
}
Optional<List<String>> getToolArgsList() {
// If "-PtoolArgs=..." is present in the command line, use it to set the args,
// otherwise use the default flag, which is "--args" to set the args.
def toolArgs = rootProject.findProperty("toolArgs")
if (toolArgs != null) {
def delimiter = '|'
toolArgs += delimiter
def argsList = []
def currArg = ''
for (def i = 0; i < toolArgs.length(); i++) {
def currChar = toolArgs[i]
if (currChar != delimiter) {
currArg += currChar
} else if (i != 0 && toolArgs[i - 1] == '\\') {
currArg = currArg.substring(0, currArg.length() - 1) + currChar
} else {
argsList.add(currArg)
currArg = ''
}
}
return Optional.of(argsList)
}
return Optional.empty()
}
// To run the nomulus tools with these command line tokens:
// "--foo", "bar baz", "--qux=quz"
// gradle registryTool --args="--foo 'bar baz' --qux=quz"
// or:
// gradle registryTool --PtoolArgs="--foo|bar baz|--qux=quz"
// Note that the delimiting pipe can be backslash escaped if it is part of a
// parameter.
ext.createToolTask = {
taskName,
mainClass,
sourceSet = sourceSets.main ->
project.tasks.create(taskName, JavaExec) {
classpath = sourceSet.runtimeClasspath
main = mainClass
doFirst {
getToolArgsList().ifPresent {
args it
}
}
}
}
createToolTask('registryTool', 'google.registry.tools.RegistryTool')
createToolTask(
'devTool',
'google.registry.tools.DevTool',
sourceSets.nonprod)
createToolTask(
'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
project.tasks.create('initSqlPipeline', JavaExec) {
main = 'google.registry.beam.initsql.InitSqlPipeline'
doFirst {
getToolArgsList().ifPresent {
args it
}
def isDirectRunner =
args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
// The dependency containing DirectRunner is intentionally excluded from the
// production binary, so that it won't be chosen by mistake: we definitely do
// not want to use it for the real jobs, yet DirectRunner is the default if
// the user forgets to override it.
// DirectRunner is required for tests and is already on testRuntimeClasspath.
// For simplicity, we add testRuntimeClasspath to this task's classpath instead
// of defining a new configuration just for the DirectRunner dependency.
classpath =
isDirectRunner
? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
: sourceSets.main.runtimeClasspath
}
}
// Caller must provide projectId, GCP region, runner, and the kinds to delete
// (comma-separated kind names or '*' for all). E.g.:
// nom_build :core:bulkDeleteDatastore --args="--project=domain-registry-crash \
// --region=us-central1 --runner=DataflowRunner --kindsToDelete=*"
createToolTask(
'bulkDeleteDatastore',
'google.registry.beam.datastore.BulkDeleteDatastorePipeline')
project.tasks.create('generateSqlSchema', JavaExec) {
classpath = sourceSets.nonprod.runtimeClasspath
main = 'google.registry.tools.DevTool'
args = [
'-e', 'alpha',
'generate_sql_schema', '--start_postgresql', '-o',
"${rootProject.projectRootDir}/db/src/main/resources/sql/schema/" +
"db-schema.sql.generated"
]
}
task generateGoldenImages(type: FilteringTest) {
tests = ["**/webdriver/*"]
// Sets the maximum number of test executors that may exist at the same time.
maxParallelForks 5
systemProperty 'test.screenshot.dir', screenshotsForGoldensDir
systemProperty 'test.screenshot.runAllAttempts', 'true'
systemProperty 'test.screenshot.maxAttempts', '5'
doFirst {
new File(screenshotsForGoldensDir).deleteDir()
}
}
generateGoldenImages.finalizedBy(findGoldenImages)
createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
// Build the Uber jar shared by all flex-template based BEAM pipelines.
// This packages more code and dependency than necessary. However, without
// restructuring the source tree it is difficult to generate leaner jars.
createUberJar(
'beam_pipeline_common',
'beam_pipeline_common',
'')
// Create beam staging task if environment is alpha or crash.
// All other environments use formally released pipelines through CloudBuild.
//
// User should install gcloud and login to GCP before invoking this tasks.
if (environment in ['alpha', 'crash']) {
def pipelines = [
[
mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
metaData: 'google/registry/beam/init_sql_pipeline_metadata.json'
],
[
mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
],
[
mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
metaData: 'google/registry/beam/spec11_pipeline_metadata.json'
],
[
mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
metaData: 'google/registry/beam/invoicing_pipeline_metadata.json'
],
]
project.tasks.create("stage_beam_pipelines") {
doLast {
pipelines.each {
def mainClass = it['mainClass']
def metaData = it['metaData']
def pipelineName = CaseFormat.UPPER_CAMEL.to(
CaseFormat.LOWER_UNDERSCORE,
mainClass.substring(mainClass.lastIndexOf('.') + 1))
def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath
def command = "\
gcloud dataflow flex-template build \
gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
--image-gcr-path ${imageName}:live \
--sdk-language JAVA \
--flex-template-base-image JAVA11 \
--metadata-file ${projectDir}/src/main/resources/${metaData} \
--jar ${uberJarName} \
--env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
--project ${gcpProject}".toString()
rootProject.ext.execInBash(command, '/tmp')
}
}
}.dependsOn(tasks.beam_pipeline_common)
}
// A jar with classes and resources from main sourceSet, excluding internal
// data. See comments on configurations.nomulus_test above for details.
// TODO(weiminyu): release process should build this using the public repo to eliminate the need
// for excludes.
task nomulusFossJar (type: Jar) {
archiveBaseName = 'nomulus'
archiveClassifier = 'public'
from (project.sourceSets.main.output) {
exclude 'google/registry/config/files/**'
exclude 'google/registry/proxy/config/**'
}
from (project.sourceSets.main.output) {
include 'google/registry/config/files/default-config.yaml'
include 'google/registry/config/files/nomulus-config-unittest.yaml'
}
}
// An UberJar of registry test classes, resources and all dependencies.
// See comments on configurations.nomulus_test above for details.
createUberJar(
'testUberJar',
'nomulus-tests',
'',
[project.configurations.testRuntimeClasspath],
[project.sourceSets.test.output],
[
// Exclude SQL schema, which is a test dependency.
'sql/flyway/**',
// ShadowJar includes java source files when used on
// sourceSets.test.output.
'**/*.java'
])
tasks.testUberJar {
archiveClassifier = 'alldeps'
}
artifacts {
nomulus_test nomulusFossJar
nomulus_test testUberJar
}
publishing {
repositories {
maven {
url project.publish_repo
}
}
publications {
nomulusTestsPublication(MavenPublication) {
groupId 'google.registry'
artifactId 'nomulus_test'
version project.nomulus_version
artifact nomulusFossJar
artifact testUberJar
}
}
}
task buildToolImage(dependsOn: nomulus, type: Exec) {
commandLine 'docker', 'build', '-t', 'nomulus-tool', '.'
}
task generateInitSqlPipelineGraph(type: FilteringTest) {
tests = ['InitSqlPipelineGraphTest.createPipeline_compareGraph']
ignoreFailures = true
}
task updateInitSqlPipelineGraph(type: Copy) {
def graphRelativePath = 'google/registry/beam/initsql/'
from ("${projectDir}/build/resources/test/${graphRelativePath}") {
include 'pipeline_curr.dot'
rename 'curr', 'golden'
}
into "src/test/resources/${graphRelativePath}"
dependsOn generateInitSqlPipelineGraph
doLast {
if (com.google.common.base.Strings.isNullOrEmpty(project.dot_path)) {
getLogger().info('Property dot_path is null. Not creating image for pipeline graph.')
}
def dotPath = project.dot_path
if (!new File(dotPath).exists()) {
throw new RuntimeException(
"""\
${dotPath} not found. Make sure graphviz is installed
and the dot_path property is set correctly."""
.stripIndent())
}
def goldenGraph = "src/test/resources/${graphRelativePath}/pipeline_golden.dot"
def goldenImage = "src/test/resources/${graphRelativePath}/pipeline_golden.png"
def cmd = "${dotPath} -Tpng -o \"${goldenImage}\" \"${goldenGraph}\""
try {
rootProject.ext.execInBash(cmd, projectDir)
} catch (Throwable throwable) {
throw new RuntimeException(
"""\
Failed to generate golden image with command ${cmd}
Error: ${throwable.getMessage()}
""")
}
}
}
// Build the devtool jar.
createUberJar(
'devtool',
'devtool',
'google.registry.tools.DevTool',
[ project.configurations.nonprodRuntimeClasspath ],
[ project.sourceSets.nonprod.output, sourceSets.main.output ],
[ '**/*.java' ])
artifacts {
devtool devtool
}
task copyJsFilesForTestServer(dependsOn: assemble, type: Copy) {
// Unfortunately the test server relies on having some compiled JS/CSS
// in place, so copy it over here
from "${resourcesDir}/google/registry/ui/"
include '**/*.js'
include '**/*.css'
into "${project.projectDir}/src/main/resources/google/registry/ui/"
}
task runTestServer(dependsOn: copyJsFilesForTestServer, type: JavaExec) {
main = 'google.registry.server.RegistryTestServerMain'
classpath = sourceSets.test.runtimeClasspath
}
/**
* We have to break out the test suites because some of the tests conflict
* with one another, but unfortunately this breaks the "--tests" flag. The
* --tests flag only applies to the task named on the command line (usually
* just "test"), not for all tasks of type "Test".
*
* As a better solution, FilteringTest sets testNameIncludePatterns (the
* internal property that --tests sets) from the value of the "testFilter"
* property, allowing us to filter across all the tests in core without
* explicitly specifying a test task or causing errors because there are no
* matching tests in the main task.
*
* To use it, define "testFilter" to be a comma-separated collection of class
* names (wildcards are allowed):
*
* ./gradlew test -P testFilter=*.FooBar,google.registry.tools.ShellCommandTest
*/
class FilteringTest extends Test {
FilteringTest() {
useJUnitPlatform();
}
private void applyTestFilter() {
if (project.testFilter) {
testNameIncludePatterns = project.testFilter.split(',')
// By default, gradle test tasks will produce a failure if no tests
// match the include/exclude/filter rules. Since test filtering allows us
// to select a set of tests from a particular task, we don't want this
// behavior.
filter.failOnNoMatchingTests = false
}
}
/**
* Set to false if you also want to include TestCase and TestSuite classes.
*
* <p>Must be defined before "test", if at all.
*/
boolean excludeTestCases = true
void setTests(List<String> tests) {
// Common exclude pattern. See README in parent directory for explanation.
if (excludeTestCases) {
exclude "**/*TestCase.*", "**/*TestSuite.*"
}
include tests
applyTestFilter()
}
/**
* Include all of the tests (except Test{Case,TestSuite}). This actually
* doesn't explicitly "include" anything, in which cast the Test class tries
* to include everything that is not explicitly excluded.
*/
void includeAllTests() {
exclude "**/*TestCase.*", "**/*TestSuite.*"
applyTestFilter()
}
}
task fragileTest(type: FilteringTest) {
// Common exclude pattern. See README in parent directory for explanation.
tests = fragileTestPatterns
if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
exclude dockerIncompatibleTestPatterns
}
// Run every test class in a freshly started process.
forkEvery 1
doFirst {
new File(screenshotsDir).deleteDir()
}
}
task outcastTest(type: FilteringTest) {
tests = outcastTestPatterns
// Sets the maximum number of test executors that may exist at the same time.
// Note that this number appears to contribute to NoClassDefFoundError
// exceptions on certain machines and distros. The root cause is unclear.
// Try reducing this number if you experience similar problems.
maxParallelForks 3
}
// Dedicated test suite for schema-dependent tests.
task sqlIntegrationTest(type: FilteringTest) {
// TestSuite still requires a JUnit 4 runner, which knows how to handle JUnit 5 tests.
// Here we need to override parent's choice of JUnit 5. If changing this, remember to
// change :integration:sqlIntegrationTest too.
useJUnit()
excludeTestCases = false
tests = ['google/registry/schema/integration/SqlIntegrationTestSuite.*']
}
// Verifies that RegistryTool can be instantiated:
// - All dependencies are packaged in nomulus.jar
// - JPA setup succeeds.
task registryToolIntegrationTest(dependsOn: nomulus, type: FilteringTest) {
tests = ['google/registry/tools/RegistryToolTest.*']
testClassesDirs = sourceSets.test.output.classesDirs
classpath = nomulus.outputs.files.plus(configurations.testRuntimeClasspath)
.plus(files(testClassesDirs))
}
task standardTest(type: FilteringTest) {
includeAllTests()
exclude fragileTestPatterns
exclude outcastTestPatterns
// See SqlIntegrationTestSuite.java
exclude '**/*BeforeSuiteTest.*', '**/*AfterSuiteTest.*'
if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") {
exclude dockerIncompatibleTestPatterns
}
// Run every test class in its own process.
// Uncomment to unblock build while troubleshooting inexplicable test errors.
// This setting makes the build take 35 minutes, without it it takes about 10.
// forkEvery 1
// Sets the maximum number of test executors that may exist at the same time.
// Also, Gradle executes tests in 1 thread and some of our test infrastructures
// depend on that, e.g. DualDatabaseTestInvocationContextProvider injects
// different implementation of TransactionManager into TransactionManagerFactory.
maxParallelForks 5
systemProperty 'test.projectRoot', rootProject.projectRootDir
systemProperty 'test.resourcesDir', resourcesDir
}
test {
// Don't run any tests from this task, all testing gets done in the
// FilteringTest tasks.
exclude "**"
// TODO(weiminyu): Remove dependency on sqlIntegrationTest
}.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
project.build.dependsOn devtool
project.build.dependsOn buildToolImage
project.build.dependsOn ':stage'