// Copyright 2019 The Nomulus Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. plugins { id 'java-library' } // Path to code generated by ad hoc tasks in this project. A separate path is // used for easy inspection. def generatedDir = "${project.buildDir}/generated/source/custom/main" def resourcesDir = "${project.buildDir}/resources/main" def screenshotsDir = "${project.buildDir}/screenshots" def screenshotsForGoldensDir = "${project.buildDir}/screenshots_for_goldens" def newGoldensDir = "${project.buildDir}/new_golden_images" def goldensDir = "${javaTestDir}/google/registry/webdriver/goldens/chrome-linux" def jsDir = "${project.projectDir}/src/main/javascript" // Tests that conflict with (mostly unidentified) members of the main test // suite. It is unclear if they are offenders (i.e., those that pollute global // state) or victims. // TODO(weiminyu): identify cause and fix offending tests. def outcastTestPatterns = [ // Problem seems to lie with AppEngine TaskQueue for test. "google/registry/batch/DeleteContactsAndHostsActionTest.*", "google/registry/batch/RefreshDnsOnHostRenameActionTest.*", "google/registry/flows/CheckApiActionTest.*", "google/registry/flows/EppLifecycleHostTest.*", "google/registry/flows/domain/DomainCreateFlowTest.*", "google/registry/flows/domain/DomainUpdateFlowTest.*", "google/registry/tools/CreateDomainCommandTest.*", "google/registry/tools/server/CreatePremiumListActionTest.*", ] // Tests that conflict with members of both the main test suite and the // outcast suite. They seem to be affected by global states outside of // Nomulus classes, e.g., threads and objects retained by frameworks. // TODO(weiminyu): identify cause and fix offending tests. def fragileTestPatterns = [ // Problem seems to lie with AppEngine TaskQueue for test. "google/registry/cron/TldFanoutActionTest.*", // Test Datastore inexplicably aborts transaction. "google/registry/model/tmch/ClaimsListShardTest.*", // Creates large object (64MBytes), occasionally throws OOM error. "google/registry/model/server/KmsSecretRevisionTest.*" ] // Tests that fail when running Gradle in a docker container, e. g. when // building the release artifacts in Google Cloud Build. def dockerIncompatibleTestPatterns = [ // The webdriver tests start headless Chrome in a Docker container, // resulting in Docker-in-Docker complications. "google/registry/webdriver/*", // PathParameterTest includes tests which validate that file permissions are // respected. However when running in Docker the user is root by default, so // every file is read/write-able. There is no way to exclude specific test // methods, so we exclude the whole test class. "google/registry/tools/params/PathParameterTest.*" ] sourceSets { main { java { srcDirs += generatedDir // Javadoc API is deprecated and removed in Java 12. // TODO(jianglai): re-enable after migrating to the new Javadoc API if ((JavaVersion.current().majorVersion as Integer) > 11) { exclude 'google/registry/documentation/**' } } resources { exclude '**/*.xjb' } } test { java { // Javadoc API is deprecated and removed in Java 12. // TODO(jianglai): re-enable after migrating to the new Javadoc API if ((JavaVersion.current().majorVersion as Integer) > 11) { exclude 'google/registry/documentation/**' } } resources { exclude '**/*.xjb', '**/*.xsd' } } } processTestResources { exclude '**/webdriver/*' } configurations { css jaxb soy closureCompiler // Exclude non-canonical servlet-api jars. Our AppEngine deployment uses // javax.servlet:servlet-api:2.5 // For reasons we do not understand, marking the following dependencies as // compileOnly instead of compile does not exclude them from runtimeClasspath. all { // servlet-api:3.1 pulled in but not used by soy compiler exclude group: 'javax.servlet', module: 'javax.servlet-api' // Jetty's servlet-api:2.5 implementation, pulled in by other Jetty jars exclude group: 'org.mortbay.jetty', module: 'servlet-api' } } // Known issues: // - The (test/)compile/runtime labels are deprecated. We continue using these // labels due to nebula-lint. // TODO(weiminyu): switch to api/implementation labels. // See https://github.com/nebula-plugins/gradle-lint-plugin/issues/130 for // issue status. // - Nebula-lint's conflict between unused and undeclared dependency check. // If an undeclared dependency is added, the unused-dependency check will flag // it. For now we wrap affected dependency in gradleLint.ignore block. // TODO(weiminyu): drop gradleLint.ignore block when issue is fixed. // See https://github.com/nebula-plugins/gradle-lint-plugin/issues/181 for // issue status. dependencies { def deps = rootProject.dependencyMap // Custom-built objectify jar at commit ecd5165, included in Nomulus // release. implementation files( "${rootDir}/third_party/objectify/v4_1/objectify-4.1.3.jar") testImplementation project(':third_party') testRuntime files(sourceSets.test.resources.srcDirs) compile deps['com.beust:jcommander'] compile deps['com.google.api-client:google-api-client'] compile deps['com.google.monitoring-client:metrics'] compile deps['com.google.monitoring-client:stackdriver'] compile deps['com.google.api-client:google-api-client-java6'] compile deps['com.google.apis:google-api-services-admin-directory'] compile deps['com.google.apis:google-api-services-appengine'] compile deps['com.google.apis:google-api-services-bigquery'] compile deps['com.google.apis:google-api-services-cloudkms'] compile deps['com.google.apis:google-api-services-dataflow'] compile deps['com.google.apis:google-api-services-dns'] compile deps['com.google.apis:google-api-services-drive'] compile deps['com.google.apis:google-api-services-groupssettings'] compile deps['com.google.apis:google-api-services-monitoring'] compile deps['com.google.apis:google-api-services-sheets'] testCompileOnly deps['com.google.appengine:appengine-api-1.0-sdk'] testCompile deps['com.google.appengine:appengine-api-stubs'] compile deps['com.google.appengine.tools:appengine-gcs-client'] compile deps['com.google.appengine.tools:appengine-mapreduce'] compile deps['com.google.appengine.tools:appengine-pipeline'] compile deps['com.google.appengine:appengine-remote-api'] compile deps['com.google.auth:google-auth-library-credentials'] compile deps['com.google.auth:google-auth-library-oauth2-http'] compile deps['com.google.code.gson:gson'] compile deps['com.google.auto.value:auto-value-annotations'] compile deps['com.google.code.findbugs:jsr305'] compile deps['com.google.dagger:dagger'] compile deps['com.google.errorprone:error_prone_annotations'] compile deps['com.google.flogger:flogger'] runtime deps['com.google.flogger:flogger-system-backend'] compile deps['com.google.guava:guava'] gradleLint.ignore('unused-dependency') { compile deps['com.google.gwt:gwt-user'] } compile deps['com.google.http-client:google-http-client'] compile deps['com.google.http-client:google-http-client-appengine'] compile deps['com.google.http-client:google-http-client-jackson2'] compile deps['com.google.oauth-client:google-oauth-client'] compile deps['com.google.oauth-client:google-oauth-client-java6'] compile deps['com.google.oauth-client:google-oauth-client-jetty'] compile deps['com.google.re2j:re2j'] compile deps['com.google.template:soy'] compile deps['com.googlecode.json-simple:json-simple'] compile deps['com.jcraft:jsch'] testCompile deps['com.thoughtworks.qdox:qdox'] compile deps['dnsjava:dnsjava'] runtime deps['org.glassfish.jaxb:jaxb-runtime'] testCompile deps['javax.annotation:jsr250-api'] compile deps['javax.inject:javax.inject'] compile deps['javax.mail:mail'] compile deps['javax.servlet:servlet-api'] compile deps['javax.xml.bind:jaxb-api'] compile deps['jline:jline'] compile deps['joda-time:joda-time'] compile deps['org.apache.avro:avro'] testCompile deps['org.apache.beam:beam-runners-direct-java'] compile deps['org.apache.beam:beam-runners-google-cloud-dataflow-java'] compile deps['org.apache.beam:beam-sdks-java-core'] compile deps['org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core'] compile deps['org.apache.beam:beam-sdks-java-io-google-cloud-platform'] testCompile deps['org.apache.commons:commons-text'] testCompile deps['org.apache.ftpserver:ftplet-api'] testCompile deps['org.apache.ftpserver:ftpserver-core'] compile deps['org.apache.httpcomponents:httpclient'] compile deps['org.apache.httpcomponents:httpcore'] testCompile deps['org.apache.sshd:sshd-core'] testCompile deps['org.apache.sshd:sshd-scp'] testCompile deps['org.apache.sshd:sshd-sftp'] testCompile deps['org.apache.tomcat:tomcat-annotations-api'] compile deps['org.bouncycastle:bcpg-jdk15on'] testCompile deps['org.bouncycastle:bcpkix-jdk15on'] compile deps['org.bouncycastle:bcprov-jdk15on'] compile deps['org.joda:joda-money'] compile deps['org.json:json'] testCompile deps['org.mortbay.jetty:jetty'] testCompile deps['org.seleniumhq.selenium:selenium-api'] testCompile deps['org.seleniumhq.selenium:selenium-chrome-driver'] testCompile deps['org.seleniumhq.selenium:selenium-java'] testCompile deps['org.seleniumhq.selenium:selenium-remote-driver'] testCompile deps['org.testcontainers:selenium'] compile deps['xerces:xmlParserAPIs'] compile deps['xpp3:xpp3'] // Known issue: nebula-lint misses inherited dependency. compile project(':third_party') compile project(':util') // Include auto-value in compile until nebula-lint understands // annotationProcessor gradleLint.ignore('unused-dependency') { compile deps['com.google.auto.value:auto-value'] } annotationProcessor deps['com.google.auto.value:auto-value'] testAnnotationProcessor deps['com.google.auto.value:auto-value'] annotationProcessor deps['com.google.dagger:dagger-compiler'] testAnnotationProcessor deps['com.google.dagger:dagger-compiler'] testCompile deps['com.google.appengine:appengine-testing'] testCompile deps['com.google.guava:guava-testlib'] testCompile deps['com.google.monitoring-client:contrib'] testCompile deps['com.google.truth:truth'] testCompile deps['com.google.truth.extensions:truth-java8-extension'] testCompile deps['org.hamcrest:hamcrest-all'] testCompile deps['org.hamcrest:hamcrest-core'] testCompile deps['org.hamcrest:hamcrest-library'] compile deps['org.hibernate:hibernate-core'] testCompile deps['junit:junit'] testCompile deps['org.mockito:mockito-core'] runtime deps['org.postgresql:postgresql'] // Indirect dependency found by undeclared-dependency check. Such // dependencies should go after all other compile and testCompile // dependencies to avoid overriding them accidentally. compile deps['com.google.oauth-client:google-oauth-client-java6'] // Dependencies needed for jaxb compilation. // Use jaxb 2.2.11 because 2.3 is known to break the Ant task we use. // TODO: upgrade jaxb versions to 2.4.0, already in beta by Sept 2018 jaxb deps['javax.xml.bind:jaxb-api'] jaxb deps['com.sun.activation:javax.activation'] jaxb deps['com.sun.xml.bind:jaxb-xjc'] jaxb deps['com.sun.xml.bind:jaxb-impl'] jaxb deps['com.sun.xml.bind:jaxb-osgi'] // Dependency needed for soy to java compilation. soy deps['com.google.template:soy'] // Dependencies needed for compiling stylesheets to javascript css deps['com.google.closure-stylesheets:closure-stylesheets'] css deps['args4j:args4j'] // Tool dependencies. used for doc generation. compile files("${System.properties['java.home']}/../lib/tools.jar") closureCompiler deps['com.google.javascript:closure-compiler'] } task jaxbToJava { def xsdFilesDir = "${javaDir}/google/registry/xml/xsd" def bindingsFile = "${javaDir}/google/registry/xjc/bindings.xjb" def pkgInfoGenerator = "${javaDir}/google/registry/xjc/make_pkginfo.sh" def pkgInfoTemplate = "${javaDir}/google/registry/xjc/package-info.java.in" def outputDir = "${generatedDir}/google/registry/xjc" inputs.dir xsdFilesDir inputs.files bindingsFile, pkgInfoTemplate, pkgInfoGenerator outputs.dir outputDir doLast { file(generatedDir).mkdirs() // Temp dir to hold schema and bindings files. Files must be in the same // directory because the bindings (.xjb) file does not declare relative // paths to schema (.xsd) files. def xjcTempSourceDir = file("${temporaryDir}/xjc") xjcTempSourceDir.mkdirs() ant.copy( todir: "${xjcTempSourceDir}", overwrite: true) { fileSet( dir: xsdFilesDir, includes: '**.xsd') } ant.copy( todir: "${xjcTempSourceDir}", overwrite: true, file: bindingsFile) ant.taskdef( name: 'xjc', classname: 'com.sun.tools.xjc.XJCTask', classpath: configurations.jaxb.asPath) ant.xjc( destdir: "${generatedDir}", binding: "${xjcTempSourceDir}/bindings.xjb", removeOldOutput: 'yes', extension: 'true') { project.fileTree( dir: new File("$xjcTempSourceDir"), include: ['**/*.xsd']) .addToAntBuilder(ant, 'schema', FileCollection.AntType.FileSet) // -npa: do not generate package-info.java files. They will be generated // below. arg(line: '-npa -quiet -extension') } exec { workingDir "${generatedDir}" executable pkgInfoGenerator args pkgInfoTemplate, outputDir } } } task soyToJava { // Relative paths of soy directories. def spec11SoyDir = "google/registry/reporting/spec11/soy" def toolsSoyDir = "google/registry/tools/soy" def uiSoyDir = "google/registry/ui/soy" def registrarSoyDir = "google/registry/ui/soy/registrar" def soyRelativeDirs = [ spec11SoyDir, toolsSoyDir, uiSoyDir, registrarSoyDir, ] soyRelativeDirs.each { inputs.dir "${resourcesSourceDir}/${it}" outputs.dir "${generatedDir}/${it}" } ext.soyToJava = { javaPackage, outputDirectory, soyFiles -> javaexec { main = "com.google.template.soy.SoyParseInfoGenerator" classpath configurations.soy args "--javaPackage", "${javaPackage}", "--outputDirectory", "${outputDirectory}", "--javaClassNameSource", "filename", "--allowExternalCalls", "true", "--srcs", "${soyFiles.join(',')}" } } doLast { soyToJava('google.registry.tools.soy', "${generatedDir}/${toolsSoyDir}", fileTree( dir: "${resourcesSourceDir}/${toolsSoyDir}", include: ['**/*.soy'])) soyToJava('google.registry.ui.soy.registrar', "${generatedDir}/${registrarSoyDir}", fileTree( dir: "${resourcesSourceDir}/${registrarSoyDir}", include: ['**/*.soy'])) soyToJava('google.registry.ui.soy', "${generatedDir}/${uiSoyDir}", files { file("${resourcesSourceDir}/${uiSoyDir}").listFiles() }.filter { it.name.endsWith(".soy") }) soyToJava('google.registry.reporting.spec11.soy', "${generatedDir}/${spec11SoyDir}", fileTree( dir: "${resourcesSourceDir}/${spec11SoyDir}", include: ['**/*.soy'])) } } task soyToJS { def rootSoyDirectory = "${resourcesSourceDir}/google/registry/ui/soy" def outputSoyDirectory = "${generatedDir}/google/registry/ui/soy" inputs.dir rootSoyDirectory outputs.dir outputSoyDirectory ext.soyToJS = { outputDirectory, soyFiles , deps-> javaexec { main = "com.google.template.soy.SoyToJsSrcCompiler" classpath configurations.soy args "--outputPathFormat", "${outputDirectory}/{INPUT_FILE_NAME}.js", "--allowExternalCalls", "false", "--srcs", "${soyFiles.join(',')}", "--shouldProvideRequireSoyNamespaces", "true", "--compileTimeGlobalsFile", "${resourcesSourceDir}/google/registry/ui/globals.txt", "--deps", "${deps.join(',')}" } } doLast { def rootSoyFiles = fileTree( dir: "${rootSoyDirectory}", include: ['*.soy']) soyToJS("${outputSoyDirectory}", rootSoyFiles, "") soyToJS("${outputSoyDirectory}/registrar", files { file("${rootSoyDirectory}/registrar").listFiles() }.filter { it.name.endsWith(".soy") }, rootSoyFiles) } } task stylesheetsToJavascript { def cssSourceDir = "${jsDir}/google/registry/ui/css" def outputDir = "${resourcesDir}/google/registry/ui/css" inputs.dir cssSourceDir outputs.dir outputDir ext.cssCompile = { outputName, debug, cssFiles -> javaexec { main = "com.google.common.css.compiler.commandline.ClosureCommandLineCompiler" classpath configurations.css def argsBuffer = [ "--output-file", "${outputName}.css", "--output-source-map", "${outputName}.css.map", "--input-orientation", "LTR", "--output-orientation", "NOCHANGE", "--output-renaming-map", "${outputName}.css.js", "--output-renaming-map-format", "CLOSURE_COMPILED_SPLIT_HYPHENS" ] if (debug) { argsBuffer.addAll(["--rename", "DEBUG", "--pretty-print"]) } else { argsBuffer.addAll(["--rename", "CLOSURE"]) } argsBuffer.addAll(cssFiles) args argsBuffer } } doLast { file("${outputDir}").mkdirs() def ignoredFiles = ["demo_css.css", "registrar_imports_raw.css"] def sourceFiles = [] // include all CSS files that we find except for the ones explicitly ignored fileTree(cssSourceDir).each { if (it.name.endsWith(".css") && !ignoredFiles.contains(it.name)) { sourceFiles << (cssSourceDir + "/" + it.name) } } // The css files have to be passed to the compiler in alphabetic order to // avoid some flaky style issues sourceFiles.sort() cssCompile("${outputDir}/registrar_bin", false, sourceFiles) cssCompile("${outputDir}/registrar_dbg", true, sourceFiles) } } task compileProdJS(type: JavaExec) { def outputDir = "${resourcesDir}/google/registry/ui" def nodeModulesDir = "${rootDir}/node_modules" def cssSourceDir = "${resourcesDir}/google/registry/ui/css" def jsSourceDir = "${jsDir}/google/registry/ui/js" def externsDir = "${jsDir}/google/registry/ui/externs" def soySourceDir = "${generatedDir}/google/registry/ui/soy" [nodeModulesDir, cssSourceDir, jsSourceDir, externsDir, soySourceDir].each { inputs.dir "${it}" } outputs.dir outputDir classpath configurations.closureCompiler main = 'com.google.javascript.jscomp.CommandLineRunner' def closureArgs = [] closureArgs << "--js_output_file=${outputDir}/registrar_bin.js" // sourcemap-related configuration closureArgs << "--create_source_map=${outputDir}/registrar_bin.js.map" closureArgs << "--source_map_include_content=true" closureArgs << "--source_map_location_mapping=${rootDir}/|" closureArgs << "--output_wrapper=\"%output% //# sourceMappingURL=registrar_bin.js.map\"" // compilation options closureArgs << "--compilation_level=ADVANCED" closureArgs << "--entry_point=goog:registry.registrar.main" closureArgs << "--generate_exports" // manually include all the required js files closureArgs << "--js=${nodeModulesDir}/google-closure-library/**.js" closureArgs << "--js=${nodeModulesDir}/soyutils_usegoog.js" closureArgs << "--js=${cssSourceDir}/registrar_bin.css.js" closureArgs << "--js=${jsSourceDir}/**.js" // TODO(shicong) Verify the compiled JS file works in Alpha closureArgs << "--js=${externsDir}/json.js" closureArgs << "--js=${soySourceDir}/**.js" args closureArgs } compileJava.dependsOn jaxbToJava compileJava.dependsOn soyToJava compileJava.dependsOn compileProdJS // stylesheetsToJavascript must happen after processResources, which wipes the // resources folder before copying data into it. stylesheetsToJavascript.dependsOn processResources classes.dependsOn stylesheetsToJavascript compileProdJS.dependsOn stylesheetsToJavascript compileProdJS.dependsOn rootProject.npmInstall compileProdJS.dependsOn processResources compileProdJS.dependsOn processTestResources compileProdJS.dependsOn soyToJS assemble.dependsOn compileProdJS task karmaTest(type: Exec) { dependsOn ':npmInstall' workingDir rootProject.projectDir executable 'node_modules/karma/bin/karma' args('start') } test.dependsOn karmaTest // Make testing artifacts available to be depended up on by other projects. // TODO: factor out google.registry.testing to be a separate project. task testJar(type: Jar) { classifier = 'test' from sourceSets.test.output } artifacts { testRuntime testJar } task fragileTest(type: Test) { // Common exclude pattern. See README in parent directory for explanation. exclude "**/*TestCase.*", "**/*TestSuite.*" include fragileTestPatterns // Run every test class in a freshly started process. forkEvery 1 // Uncomment to see test outputs in stdout. //testLogging.showStandardStreams = true } task outcastTest(type: Test) { // Common exclude pattern. See README in parent directory for explanation. exclude "**/*TestCase.*", "**/*TestSuite.*" include outcastTestPatterns // Sets the maximum number of test executors that may exist at the same time. maxParallelForks 5 } task findGoldenImages(type: JavaExec) { classpath = sourceSets.test.runtimeClasspath main = 'google.registry.webdriver.GoldenImageFinder' def arguments = [] arguments << "--screenshots_for_goldens_dir=${screenshotsForGoldensDir}" arguments << "--new_goldens_dir=${newGoldensDir}" arguments << "--existing_goldens_dir=${goldensDir}" if (rootProject.findProperty("overrideExistingGoldens") == "true") { arguments << "--override_existing_goldens=true" } args arguments } task generateGoldenImages(type: Test) { // Common exclude pattern. See README in parent directory for explanation. exclude "**/*TestCase.*", "**/*TestSuite.*" include "**/webdriver/*" // Sets the maximum number of test executors that may exist at the same time. maxParallelForks 5 systemProperty 'test.screenshot.dir', screenshotsForGoldensDir systemProperty 'test.screenshot.runAllAttempts', 'true' systemProperty 'test.screenshot.maxAttempts', '5' doFirst { new File(screenshotsForGoldensDir).deleteDir() } } generateGoldenImages.finalizedBy(findGoldenImages) task flowDocsTool(type: JavaExec) { systemProperty 'test.projectRoot', rootProject.projectRootDir systemProperty 'test.resourcesDir', resourcesDir classpath = sourceSets.main.runtimeClasspath main = 'google.registry.documentation.FlowDocumentationTool' def arguments = [] if (rootProject.flowDocsFile) { arguments << "--output_file=${rootProject.flowDocsFile}" } else { arguments << "--output_file=${rootProject.projectRootDir}/docs/flows.md" } args arguments } test { // Common exclude pattern. See README in parent directory for explanation. exclude "**/*TestCase.*", "**/*TestSuite.*" exclude fragileTestPatterns exclude outcastTestPatterns if (rootProject.findProperty("skipDockerIncompatibleTests") == "true") { exclude dockerIncompatibleTestPatterns } // Run every test class in its own process. // Uncomment to unblock build while troubleshooting inexplicable test errors. // This setting makes the build take 35 minutes, without it it takes about 10. // forkEvery 1 // Sets the maximum number of test executors that may exist at the same time. maxParallelForks 5 systemProperty 'test.projectRoot', rootProject.projectRootDir systemProperty 'test.resourcesDir', resourcesDir doFirst { new File(screenshotsDir).deleteDir() } }.dependsOn(fragileTest, outcastTest) createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool') project.nomulus.dependsOn project(':third_party').jar task buildToolImage(dependsOn: nomulus, type: Exec) { commandLine 'docker', 'build', '-t', 'nomulus-tool', '.' } task copyJsFilesForTestServer(dependsOn: assemble, type: Copy) { // Unfortunately the test server relies on having some compiled JS/CSS // in place, so copy it over here from "${resourcesDir}/google/registry/ui/" include '**/*.js' include '**/*.css' into "${project.projectDir}/src/main/resources/google/registry/ui/" } task runTestServer(dependsOn: copyJsFilesForTestServer, type: JavaExec) { main = 'google.registry.server.RegistryTestServerMain' classpath = sourceSets.test.runtimeClasspath } project.build.dependsOn buildToolImage project.build.dependsOn ':stage'