diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 000000000..5f328ec1f --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,6 @@ +docs/** +python/** +bazel-*/** +gradle/**/build/** +gradle/**/WEB-INF/** +gradle/.*/** diff --git a/.gitignore b/.gitignore index d379d1566..7d0db63f6 100644 --- a/.gitignore +++ b/.gitignore @@ -80,10 +80,9 @@ autogenerated/ ###################################################################### # Gradle Ignores +# We don't want to ignore the gradle jar files +!/gradle/gradle/wrapper/**/*.jar + /gradle/.gradle -/gradle/gradle -/gradle/gradlew -/gradle/gradlew.bat /gradle/**/WEB-INF /gradle/**/build - diff --git a/.travis.yml b/.travis.yml index 394f23414..879ab3d2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,13 +33,8 @@ env: # quota) TERM=dumb -install: - # Install a specific gradle version first, default gradle can't deal with - # our gradle build scripts. - - wget http://services.gradle.org/distributions/gradle-4.10.2-bin.zip && unzip gradle-4.10.2-bin.zip - # Specialize gradle build to use an up-to-date gradle and the /gradle # directory. # The "travis_wait 45" lets our build spend up to 45 minutes without writing # output, instead of the default 10. -script: cd gradle && travis_wait 45 ../gradle-4.10.2/bin/gradle build +script: cd gradle && chmod 755 ./gradlew && travis_wait 45 ./gradlew build diff --git a/README.md b/README.md index c0b669c20..d69b9c31b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Nomulus -![Build Status](https://storage.googleapis.com/domain-registry-github-build-status/github-ci-status.png) +| Bazel | Gradle | +|-------|--------| +|![Build Status](https://storage.googleapis.com/domain-registry-github-build-status/github-ci-status.png)|[![Build Status](https://travis-ci.org/google/nomulus.svg?branch=master)](https://travis-ci.org/google/nomulus)| ![Nomulus logo](./nomulus-logo.png) diff --git a/WORKSPACE b/WORKSPACE index c5ab5f178..3e2d8b42c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,7 @@ workspace(name = "domain_registry") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + # https://github.com/bazelbuild/rules_closure/releases/tag/0.8.0 http_archive( name = "io_bazel_rules_closure", @@ -14,6 +16,7 @@ load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories") closure_repositories( omit_com_google_auto_factory = True, + omit_com_google_protobuf = True, omit_com_google_code_findbugs_jsr305 = True, omit_com_google_guava = True, omit_com_ibm_icu_icu4j = True, @@ -26,10 +29,11 @@ load("//java/google/registry:repositories.bzl", "domain_registry_repositories") domain_registry_repositories() # Setup docker bazel rules -git_repository( +http_archive( name = "io_bazel_rules_docker", - remote = "https://github.com/bazelbuild/rules_docker.git", - tag = "v0.4.0", + sha256 = "29d109605e0d6f9c892584f07275b8c9260803bf0c6fcb7de2623b2bedc910bd", + strip_prefix = "rules_docker-0.5.1", + urls = ["https://github.com/bazelbuild/rules_docker/archive/v0.5.1.tar.gz"], ) load( @@ -38,11 +42,14 @@ load( container_repositories = "repositories", ) +# This is NOT needed when going through the language lang_image +# "repositories" function(s). container_repositories() container_pull( name = "java_base", registry = "gcr.io", repository = "distroless/java", - digest = "sha256:780ee786a774a25a4485f491b3e0a21f7faed01864640af7cebec63c46a0845a", + # 'tag' is also supported, but digest is encouraged for reproducibility. + digest = "sha256:8c1769cb253bdecc257470f7fba05446a55b70805fa686f227a11655a90dfe9e", ) diff --git a/cloudbuild-nomulus.yaml b/cloudbuild-nomulus.yaml new file mode 100644 index 000000000..9f8ac19be --- /dev/null +++ b/cloudbuild-nomulus.yaml @@ -0,0 +1,53 @@ +# To run the build locally, install cloud-build-local first. +# See: https://cloud.google.com/cloud-build/docs/build-debug-locally +# Then run: +# cloud-build-local --config=cloudbuild-nomulus.yaml --dryrun=false --substitutions TAG_NAME=[TAG] . +# This will create a docker image named gcr.io/[PROJECT_ID]/proxy:[TAG] locally. +# The PROJECT_ID is the current project name that gcloud uses. +# You can add "--push true" to have the image pushed to GCR. +# +# To manually trigger a build on GCB, run: +# gcloud builds submit --config cloudbuild-nomulus.yaml --substitutions TAG_NAME=[TAG] . +# +# To trigger a build automatically, follow the instructions below and add a trigger: +# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds +steps: +# Set permissions correctly. Not sure why it is necessary, but it is. +- name: 'alpine' + args: ['chown', '-R', 'root:root', '.'] +- name: 'alpine' + args: ['chmod', '-R', '777', '.'] +# Clone the private repo and merge its contents. +- name: 'gcr.io/cloud-builders/gcloud' + args: ['source', 'repos', 'clone', 'nomulus-config'] +- name: 'alpine' + args: ['sh', '-c', 'cp -r nomulus-config/* .'] +# Build the deployment files. +- name: 'google/cloud-sdk' + args: ['./gradlew', 'stage', '-x', 'autoLintGradle'] + dir: 'gradle' +# Tar the deployment files as we cannot upload directories to GCS. +- name: 'alpine' + args: ['tar', 'cvf', '../../../default.tar', '.'] + dir: 'gradle/services/default/build/staged-app' +- name: 'alpine' + args: ['tar', 'cvf', '../../../pubapi.tar', '.'] + dir: 'gradle/services/pubapi/build/staged-app' +- name: 'alpine' + args: ['tar', 'cvf', '../../../backend.tar', '.'] + dir: 'gradle/services/backend/build/staged-app' +- name: 'alpine' + args: ['tar', 'cvf', '../../../tools.tar', '.'] + dir: 'gradle/services/tools/build/staged-app' +# Tar files to upload to GCS. +artifacts: + objects: + location: 'gs://${PROJECT_ID}-deploy/${TAG_NAME}' + paths: + - 'gradle/services/default.tar' + - 'gradle/services/pubapi.tar' + - 'gradle/services/backend.tar' + - 'gradle/services/tools.tar' +timeout: 3600s +options: + machineType: 'N1_HIGHCPU_8' diff --git a/cloudbuild-proxy.yaml b/cloudbuild-proxy.yaml new file mode 100644 index 000000000..a95c87d3d --- /dev/null +++ b/cloudbuild-proxy.yaml @@ -0,0 +1,37 @@ +# To run the build locally, install cloud-build-local first. +# See: https://cloud.google.com/cloud-build/docs/build-debug-locally +# Then run: +# cloud-build-local --config=cloudbuild-proxy.yaml --dryrun=false --substitutions TAG_NAME=[TAG] . +# This will create a docker image named gcr.io/[PROJECT_ID]/proxy:[TAG] locally. +# The PROJECT_ID is the current project name that gcloud uses. +# You can add "--push true" to have the image pushed to GCR. +# +# To manually trigger a build on GCB, run: +# gcloud builds submit --config cloudbuild-proxy.yaml --substitutions TAG_NAME=[TAG] . +# +# To trigger a build automatically, follow the instructions below and add a trigger: +# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds +steps: +# Set permissions correctly. Not sure why it is necessary, but it is. +- name: 'alpine' + args: ['chown', '-R', 'root:root', '.'] +- name: 'alpine' + args: ['chmod', '-R', '777', '.'] +# Clone the private repo merge its contents. +- name: 'gcr.io/cloud-builders/gcloud' + args: ['source', 'repos', 'clone', 'nomulus-config'] +- name: 'alpine' + args: ['sh', '-c', 'cp -r nomulus-config/* .'] +# Build the deploy jar. +- name: 'openjdk:8-slim' + args: ['./gradlew', ':proxy:deployJar', '-x', 'autoLintGradle'] + dir: 'gradle' +# Build the docker image. +- name: 'gcr.io/cloud-builders/docker' + args: ['build', '--tag', 'gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}', '.'] + dir: 'gradle/proxy' +# Images to upload to GCR. +images: ['gcr.io/${PROJECT_ID}/proxy:${TAG_NAME}'] +timeout: 3600s +options: + machineType: 'N1_HIGHCPU_8' diff --git a/docs/flows.md b/docs/flows.md index 79bf91276..1b6e1d6a3 100644 --- a/docs/flows.md +++ b/docs/flows.md @@ -231,6 +231,8 @@ An EPP flow that updates a contact. An EPP flow that allocates a new domain resource from a domain application. +Note that this flow is only run by superusers. + ### Errors @@ -295,7 +297,7 @@ An EPP flow that creates a new application for a domain resource. * Specified extension is not implemented. * 2201 * Registrar is not authorized to access this TLD. - * Registrar must be active in order to create domains or applications. + * Registrar must be active in order to perform this operation. * 2302 * Resource with this id already exists. * This name has already been claimed by a sunrise applicant. @@ -556,7 +558,7 @@ An EPP flow that creates a new domain resource. * 2201 * Only a tool can pass a metadata extension. * Registrar is not authorized to access this TLD. - * Registrar must be active in order to create domains or applications. + * Registrar must be active in order to perform this operation. * 2302 * Resource with this id already exists. * 2303 @@ -689,6 +691,7 @@ comes in at the exact millisecond that the domain would have expired. * 2201 * The specified resource belongs to another client. * Registrar is not authorized to access this TLD. + * Registrar must be active in order to perform this operation. * 2303 * Resource with this id does not exist. * 2304 @@ -745,6 +748,7 @@ regardless of what the original expiration time was. * 2201 * The specified resource belongs to another client. * Registrar is not authorized to access this TLD. + * Registrar must be active in order to perform this operation. * 2303 * Resource with this id does not exist. * 2304 @@ -907,6 +911,7 @@ new ones with the correct approval time). * 2201 * Authorization info is required to request a transfer. * Registrar is not authorized to access this TLD. + * Registrar must be active in order to perform this operation. * 2202 * Authorization information for accessing resource is invalid. * 2300 @@ -1169,8 +1174,7 @@ An EPP flow for login. * 2103 * Specified extension is not implemented. * 2200 - * GAE user id is not allowed to login as requested registrar. - * User is not logged in as a GAE user. + * GAE User can't access the requested registrar. * Registrar certificate does not match stored certificate. * Registrar IP address is not in stored whitelist. * Registrar certificate not present. diff --git a/docs/install.md b/docs/install.md index 36b46c09c..949f78610 100644 --- a/docs/install.md +++ b/docs/install.md @@ -7,8 +7,8 @@ This document covers the steps necessary to download, build, and deploy Nomulus. You will need the following programs installed on your local machine: * A recent version of the [Java 8 JDK][java-jdk8]. -* [Bazel build system](http://bazel.io/) (version [0.17.2][bazel-version] - works as of 2018-10-03). +* [Bazel build system](http://bazel.io/) (version [0.21.0][bazel-version] + works as of 2018-12-20). * [Google App Engine SDK for Java][app-engine-sdk], and configure aliases to to the `gcloud` and `appcfg.sh` utilities (you'll use them a lot). * [Git](https://git-scm.com/) version control system. @@ -135,11 +135,11 @@ $ ls /path/to/app-dir/acme-registry-alpha backend default META-INF tools ``` -Now deploy the code to App Engine. +Now deploy the code to App Engine. We must provide a version string, e.g., live. ```shell $ appcfg.sh -A acme-registry-alpha --enable_jar_splitting \ - update /path/to/app-dir/acme-registry-alpha + -V live update /path/to/app-dir/acme-registry-alpha Reading application configuration data... Processing module default Oct 05, 2016 12:16:59 PM com.google.apphosting.utils.config.IndexesXmlReader readConfigXml @@ -181,4 +181,4 @@ See the [first steps tutorial](./first-steps-tutorial.md) for more information. [app-engine-sdk]: https://cloud.google.com/appengine/docs/java/download [java-jdk8]: http://www.oracle.com/technetwork/java/javase/downloads -[bazel-version]: https://github.com/bazelbuild/bazel/releases/download/0.17.2/bazel-0.17.2-installer-linux-x86_64.sh +[bazel-version]: https://github.com/bazelbuild/bazel/releases/download/0.21.0/bazel-0.21.0-installer-linux-x86_64.sh diff --git a/gradle/README.md b/gradle/README.md index 62c7a557c..57801a7cf 100644 --- a/gradle/README.md +++ b/gradle/README.md @@ -15,8 +15,6 @@ the existing Nomulus source tree. Dependencies are mostly the same as in Bazel, with a few exceptions: -* org.slf4j:slf4j-simple is added to provide a logging implementation in - tests. Bazel does not need this. * com.googlecode.java-diff-utils:diffutils is not included. Bazel needs it for Truth's equality check, but Gradle works fine without it. * jaxb 2.2.11 is used instead of 2.3 in Bazel, since the latter breaks the @@ -27,18 +25,22 @@ Dependencies are mostly the same as in Bazel, with a few exceptions: ### Notable Issues -Only single-threaded test execution is allowed, due to race condition over -global resources, such as the local Datastore instance, or updates to the System -properties. This is a new problem with Gradle, which does not provide as much -test isolation as Bazel. We are exploring solutions to this problem. - Test suites (RdeTestSuite and TmchTestSuite) are ignored to avoid duplicate execution of tests. Neither suite performs any shared test setup routine, so it -is easier to exclude the suite classes than individual test classes. +is easier to exclude the suite classes than individual test classes. This is the +reason why all test tasks in the :core project contain the exclude pattern +'"**/*TestCase.*", "**/*TestSuite.*"' -Since Gradle does not support hierarchical build files, all file sets (e.g., -resources) must be declared at the top, in root project config or the -sub-project configs. +Many Nomulus tests are not hermetic: they modify global state (e.g., the shared +local instance of Datastore) but do not clean up on completion. This becomes a +problem with Gradle. In the beginning we forced Gradle to run every test class +in a new process, and incurred heavy overheads. Since then, we have fixed some +tests, and manged to divide all tests into three suites that do not have +intra-suite conflicts. We will revisit the remaining tests soon. + +Note that it is unclear if all conflicting tests have been identified. More may +be exposed if test execution order changes, e.g., when new tests are added or +execution parallelism level changes. ## Initial Setup diff --git a/gradle/build.gradle b/gradle/build.gradle index 9c8b11b2c..68bc6796e 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -2,28 +2,284 @@ buildscript { repositories { jcenter() mavenCentral() + } - maven { - url 'https://plugins.gradle.org/m2/' - } + dependencies { + classpath 'com.google.cloud.tools:appengine-gradle-plugin:1.3.3' + classpath 'org.sonatype.aether:aether-api:1.13.1' + classpath 'org.sonatype.aether:aether-impl:1.13.1' } } -allprojects { +plugins { + id 'nebula.dependency-lock' version '7.1.0' + id 'nebula.lint' version '10.3.5' + // Config helper for annotation processors such as AutoValue and Dagger. + // Ensures that source code is generated at an appropriate location. + id 'net.ltgt.apt' version '0.19' apply false + id 'com.bmuschko.docker-java-application' version '4.0.4' apply false +} + +// Provide defaults for all of the project properties. + +// showAllOutput: boolean. If true, dump all test output during the build. +if (!project.hasProperty('showAllOutput')) { + ext.showAllOutput = 'false' +} + +// Only do linting if the build is successful. +gradleLint.autoLintAfterFailure = false + +// Paths to main and test sources. +ext.javaDir = "${rootDir}/../java" +ext.javatestsDir = "${rootDir}/../javatests" + +// Tasks to deploy/stage all App Engine services +task deploy { + group = 'deployment' + description = 'Deploys all services to App Engine.' +} + +task stage { + group = 'deployment' + description = 'Generates application directories for all services.' +} + +subprojects { + // Skip no-op project + if (project.name == 'services') return + repositories { jcenter() mavenCentral() - flatDir { - // The objectify jar that comes with Nomulus. - dirs "${rootDir}/../third_party/objectify/v4_1" + } + + def services = [':services:default', + ':services:backend', + ':services:tools', + ':services:pubapi'] + + // Set up all of the deployment projects. + if (services.contains(project.path)) { + + apply plugin: 'war' + + // Set this directory before applying the appengine plugin so that the + // plugin will recognize this as an app-engine standard app (and also + // obtains the appengine-web.xml from the correct location) + project.convention.plugins['war'].webAppDirName = + "../../../java/google/registry/env/crash/${project.name}" + + apply plugin: 'com.google.cloud.tools.appengine' + + // Get the web.xml file for the service. + war { + webInf { + from "../../../java/google/registry/env/common/${project.name}/WEB-INF" + } + } + + appengine { + deploy { + // TODO: change this to a variable. + project = 'domain-registry-crash' + } + } + + dependencies { + compile project(':core') + } + + rootProject.deploy.dependsOn appengineDeploy + rootProject.stage.dependsOn appengineStage + + // Return early, do not apply the settings below. + return + } + + apply plugin: 'java' + apply plugin: 'maven-publish' + apply plugin: 'nebula.dependency-lock' + apply plugin: 'nebula.lint' + apply plugin: 'net.ltgt.apt' + + version = '1.0' + sourceCompatibility = '1.8' + targetCompatibility = '1.8' + + compileJava {options.encoding = "UTF-8"} + + gradleLint.rules = [ + // Checks if Gradle wrapper is up-to-date + 'archaic-wrapper', + // Checks for indirect dependencies with dynamic version spec. Best + // practice calls for declaring them with specific versions. + 'undeclared-dependency', + 'unused-dependency' + // TODO(weiminyu): enable more dependency checks + ] + + publishing { + repositories { + maven { + url = project.findProperty('repositoryUrl') + } } } - // Single version across all projects for now. - version = '1.0' + ext.getDistinctResolvedArtifacts = { + def distinctResolvedArtifacts = [:] - // Java plugin: - apply plugin: 'java' + configurations.each { + if (!it.isCanBeResolved()) { + return + } + it.resolvedConfiguration.resolvedArtifacts.each { resolvedArtifact -> + if (resolvedArtifact.id.componentIdentifier.displayName in + ['project :core', 'project :proxy', 'project :util', 'project :third_party']) { + return + } + distinctResolvedArtifacts[resolvedArtifact.id.toString()] = resolvedArtifact + } + } + + return distinctResolvedArtifacts + } + + ext.generateDependencyPublications = { + def distinctResolvedArtifacts = project.ext.getDistinctResolvedArtifacts() + + distinctResolvedArtifacts.values().eachWithIndex { resolvedArtifact, n -> + project.publishing { + publications { + "maven${n}"(MavenPublication) { + artifact(resolvedArtifact.file) { + groupId = resolvedArtifact.moduleVersion.id.group + artifactId = resolvedArtifact.moduleVersion.id.name + version = resolvedArtifact.moduleVersion.id.version + classifier = resolvedArtifact.classifier + } + } + } + } + } + } + + ext.urlExists = { url -> + def connection = (HttpURLConnection) url.openConnection() + connection.setRequestMethod("HEAD") + connection.connect() + if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) { + return true + } else { + return false + } + } + + ext.writeMetadata = { resolvedArtifact, url, gitRepositoryPath -> + def groupId = resolvedArtifact.moduleVersion.id.group + def artifactId = resolvedArtifact.moduleVersion.id.name + def version = resolvedArtifact.moduleVersion.id.version + def relativeFileName = + [groupId, artifactId, 'README.domainregistry'].join('/') + def metadataFile = new File(gitRepositoryPath, relativeFileName) + metadataFile.parentFile.mkdirs() + def writer = metadataFile.newWriter() + writer << "Name: ${artifactId}\n" + writer << "Url: ${url}\n" + writer << "Version: ${version}\n" + writer.close() + } + + // This task generates a metadata file for each resolved dependency artifact. + // The file contains the name, url and version for the artifact. + task generateDependencyMetadata { + doLast { + def distinctResolvedArtifacts = project.ext.getDistinctResolvedArtifacts() + def defaultLayout = new org.sonatype.aether.util.layout.MavenDefaultLayout() + + distinctResolvedArtifacts.values().each { resolvedArtifact -> + def artifact = new org.sonatype.aether.util.artifact.DefaultArtifact( + resolvedArtifact.id.componentIdentifier.toString()) + for (repository in project.repositories) { + def mavenRepository = (MavenArtifactRepository) repository + def repositoryUri = URI.create(mavenRepository.url.toString()) + def artifactUri = repositoryUri.resolve(defaultLayout.getPath(artifact)) + if (project.ext.urlExists(artifactUri.toURL())) { + project.ext.writeMetadata( + resolvedArtifact, + artifactUri.toURL(), + project.findProperty('privateRepository') + "/${project.name}") + break + } + } + } + } + } + + if (project.name == 'third_party') return + + // Path to code generated with annotation processors. Note that this path is + // chosen by the 'net.ltgt.apt' plugin, and may change if IDE-specific plugins + // are applied, e.g., 'idea' or 'eclipse' + def aptGeneratedDir = "${project.buildDir}/generated/source/apt/main" + def aptGeneratedTestDir = "${project.buildDir}/generated/source/apt/test" + + def commonlyExcludedResources = ['**/*.java', '**/BUILD'] + + sourceSets { + main { + java { + srcDirs = [ + project(':').javaDir, + aptGeneratedDir + ] + } + resources { + srcDirs = [ + project(':').javaDir + ] + exclude commonlyExcludedResources + } + } + test { + java { + srcDirs = [ + project(':').javatestsDir, + aptGeneratedTestDir + ] + } + resources { + srcDirs = [ + project(':').javatestsDir, + ] + exclude commonlyExcludedResources + } + } + } + + test { + testLogging.showStandardStreams = Boolean.parseBoolean(showAllOutput) + } + + if (project.name == 'core') return + + ext.relativePath = "google/registry/${project.name}" + + sourceSets.each { + it.java { + include "${project.relativePath}/" + } + it.resources { + include "${project.relativePath}/" + } + } + project(':core').sourceSets.each { + it.java { + exclude "${project.relativePath}/" + } + it.resources { + exclude "${project.relativePath}/" + } + } } - - diff --git a/gradle/core/build.gradle b/gradle/core/build.gradle index c9745215c..0182ad928 100644 --- a/gradle/core/build.gradle +++ b/gradle/core/build.gradle @@ -2,38 +2,49 @@ plugins { id 'java-library' } -def javaDir = "${rootDir}/../java" -def javatestsDir = "${rootDir}/../javatests" +// Path to code generated by ad hoc tasks in this project. A separate path is +// used for easy inspection. +def generatedDir = "${project.buildDir}/generated/source/custom/main" -def generatedDir = "${project.buildDir}/generated-sources" +// Tests that conflict with (mostly unidentified) members of the main test +// suite. It is unclear if they are offenders (i.e., those that pollute global +// state) or victims. +// TODO(weiminyu): identify cause and fix offending tests. +def outcastTestPatterns = [ + "google/registry/batch/DeleteContactsAndHostsActionTest.*", + "google/registry/batch/RefreshDnsOnHostRenameActionTest.*", + "google/registry/flows/CheckApiActionTest.*", + "google/registry/flows/EppLifecycleHostTest.*", + "google/registry/flows/domain/DomainAllocateFlowTest.*", + "google/registry/flows/domain/DomainApplicationCreateFlowTest.*", + "google/registry/flows/domain/DomainApplicationUpdateFlowTest.*", + "google/registry/flows/domain/DomainCreateFlowTest.*", + "google/registry/flows/domain/DomainUpdateFlowTest.*", + "google/registry/tools/CreateDomainCommandTest.*", + "google/registry/tools/server/CreatePremiumListActionTest.*", + // Conflicts with WhoisActionTest + "google/registry/whois/WhoisHttpActionTest.*", +] + +// Tests that conflict with members of both the main test suite and the +// outcast suite. +// TODO(weiminyu): identify cause and fix offending tests. +def fragileTestPatterns = [ + "google/registry/cron/TldFanoutActionTest.*" +] sourceSets { main { java { - srcDirs = [ - "${javaDir}", - "${generatedDir}" - ] + srcDirs += generatedDir } resources { - srcDirs = [ - "${javaDir}" - ] - exclude '**/*.java', '**/*.xjb' + exclude '**/*.xjb' } } test { - java { - srcDirs = [ - "${javatestsDir}", - "${generatedDir}" - ] - } resources { - srcDirs = [ - "${javatestsDir}" - ] - exclude '**/*.java', '**/*.xsd', '**/*.xjb' + exclude '**/*.xjb', '**/*.xsd' } } } @@ -42,165 +53,200 @@ configurations { css jaxb soy + // Label for all dependencies inherited from Bazel build but not used in + // either compile or testRuntime. However, they may be needed at runtime. + // TODO(weiminyu): identify runtime dependencies and remove the rest. + maybeRuntime } - -// Relevant canned dependency labels: -// - implementation: Dependencies to be included in release distribution. -// - compileOnly: Dependencies used at compile time only for production code. They will not be -// included in release. -// - testImplementation: Dependencies needed for testing only. +// Known issues: +// - The (test/)compile/runtime labels are deprecated. We continue using these +// labels due to nebula-lint. +// TODO(weiminyu): switch to api/implementation labels. +// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/130 for +// issue status. +// - Nebula-lint's conflict between unused and undeclared dependency check. +// If an undeclared dependency is added, the unused-dependency check will flag +// it. For now we wrap affected dependency in gradleLint.ignore block. +// TODO(weiminyu): drop gradleLint.ignore block when issue is fixed. +// See https://github.com/nebula-plugins/gradle-lint-plugin/issues/181 for +// issue status. dependencies { - implementation 'com.beust:jcommander:1.48' - implementation 'com.fasterxml.jackson.core:jackson-core:2.8.5' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.8.0' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.8.5' - implementation 'com.google.api-client:google-api-client:1.22.0' - implementation 'com.google.api-client:google-api-client-appengine:1.22.0' - implementation 'com.google.api-client:google-api-client-jackson2:1.20.0' - implementation 'com.google.monitoring-client:metrics:1.0.4' - implementation 'com.google.monitoring-client:stackdriver:1.0.4' - implementation 'com.google.api-client:google-api-client-java6:1.20.0' - implementation 'com.google.api-client:google-api-client-servlet:1.22.0' - implementation 'com.google.apis:google-api-services-admin-directory:directory_v1-rev72-1.22.0' - implementation 'com.google.apis:google-api-services-bigquery:v2-rev325-1.22.0' - implementation 'com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0' - implementation 'com.google.apis:google-api-services-cloudkms:v1-rev12-1.22.0' - implementation 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-1.22.0' - implementation 'com.google.apis:google-api-services-dataflow:v1b3-rev196-1.22.0' - implementation 'com.google.apis:google-api-services-dns:v2beta1-rev6-1.22.0' - implementation 'com.google.apis:google-api-services-drive:v2-rev160-1.19.1' - implementation 'com.google.apis:google-api-services-groupssettings:v1-rev60-1.22.0' - implementation 'com.google.apis:google-api-services-monitoring:v3-rev11-1.22.0' - implementation 'com.google.apis:google-api-services-sheets:v4-rev483-1.22.0' - implementation 'com.google.apis:google-api-services-storage:v1-rev86-1.22.0' - // TODO(b/71631624): change appengine:appengine-api-1.0-sdk to testCompileOnly after - // BillingEmailUtilsTest.java is fixed. - implementation 'com.google.appengine:appengine-api-1.0-sdk:1.9.48' - implementation 'com.google.appengine:appengine-api-labs:1.9.48' - implementation 'com.google.appengine:appengine-api-stubs:1.9.48' - implementation 'com.google.appengine.tools:appengine-gcs-client:0.6' - implementation 'com.google.appengine.tools:appengine-mapreduce:0.8.5' - implementation 'com.google.appengine.tools:appengine-pipeline:0.2.13' - implementation 'com.google.appengine:appengine-tools-sdk:1.9.48' - implementation 'com.google.auth:google-auth-library-credentials:0.7.1' - implementation 'com.google.auth:google-auth-library-oauth2-http:0.7.1' - implementation 'com.google.auto:auto-common:0.8' - implementation 'com.google.auto.factory:auto-factory:1.0-beta3' - implementation 'com.google.auto.value:auto-value-annotations:1.6.2' - implementation 'com.google.cloud.bigdataoss:gcsio:1.4.5' - implementation 'com.google.cloud.bigdataoss:util:1.4.5' - implementation 'com.google.code.findbugs:jsr305:3.0.2' - implementation 'com.google.dagger:dagger:2.15' - implementation 'com.google.dagger:dagger-producers:2.15' - implementation 'com.google.errorprone:error_prone_annotations:2.1.3' - implementation 'com.google.errorprone:javac-shaded:9-dev-r4023-3' - implementation 'com.google.flogger:flogger:0.1' - implementation 'com.google.flogger:flogger-system-backend:0.1' - implementation 'com.google.gdata:core:1.47.1' - implementation 'com.google.googlejavaformat:google-java-format:1.4' - implementation 'com.google.guava:guava:25.1-jre' - implementation 'com.google.http-client:google-http-client:1.22.0' - implementation 'com.google.http-client:google-http-client-appengine:1.22.0' - implementation 'com.google.http-client:google-http-client-jackson2:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-appengine:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-java6:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-jetty:1.22.0' - implementation 'com.google.oauth-client:google-oauth-client-servlet:1.22.0' - implementation 'com.google.protobuf:protobuf-java:2.6.0' - implementation 'com.google.re2j:re2j:1.1' - implementation 'com.google.template:soy:2018-03-14' - implementation 'com.googlecode.charts4j:charts4j:1.3' - implementation 'com.googlecode.json-simple:json-simple:1.1.1' - implementation 'com.ibm.icu:icu4j:57.1' - implementation 'com.jcraft:jsch:0.1.53' - implementation 'com.jcraft:jzlib:1.1.3' - implementation 'com.squareup:javapoet:1.8.0' - implementation 'com.squareup:javawriter:2.5.1' - implementation 'com.sun.activation:javax.activation:1.2.0' - implementation 'com.thoughtworks.paranamer:paranamer:2.7' - implementation 'commons-codec:commons-codec:1.6' - implementation 'commons-logging:commons-logging:1.1.1' - implementation 'dnsjava:dnsjava:2.1.7' - implementation 'io.netty:netty-buffer:4.1.28.Final' - implementation 'io.netty:netty-codec:4.1.28.Final' - implementation 'io.netty:netty-codec-http:4.1.28.Final' - implementation 'io.netty:netty-common:4.1.28.Final' - implementation 'io.netty:netty-handler:4.1.28.Final' - implementation 'io.netty:netty-resolver:4.1.28.Final' - implementation 'io.netty:netty-tcnative:2.0.12.Final' - implementation 'io.netty:netty-tcnative-boringssl-static:2.0.12.Final' - implementation 'io.netty:netty-transport:4.1.28.Final' - implementation 'it.unimi.dsi:fastutil:6.5.16' - implementation 'javax.annotation:jsr250-api:1.0' - implementation 'javax.inject:javax.inject:1' - implementation 'javax.mail:mail:1.4' - implementation 'javax.servlet:servlet-api:2.5' - implementation 'javax.xml.bind:jaxb-api:2.3.0' - implementation 'javax.xml.soap:javax.xml.soap-api:1.4.0' - implementation 'jline:jline:1.0' - implementation 'joda-time:joda-time:2.3' - implementation 'org.apache.avro:avro:1.8.2' - implementation 'org.apache.beam:beam-runners-direct-java:2.2.0' - implementation 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.1.0' - implementation 'org.apache.beam:beam-sdks-common-runner-api:2.1.0' - implementation 'org.apache.beam:beam-sdks-java-core:2.2.0' - implementation 'org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.1.0' - implementation 'org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.2.0' - implementation 'org.apache.commons:commons-compress:1.8.1' - implementation 'org.apache.ftpserver:ftpserver-core:1.0.6' - implementation 'org.apache.httpcomponents:httpclient:4.5.2' - implementation 'org.apache.httpcomponents:httpcore:4.4.4' - implementation 'org.apache.mina:mina-core:2.0.4' - implementation 'org.apache.sshd:sshd-core:2.0.0' - implementation 'org.apache.sshd:sshd-scp:2.0.0' - implementation 'org.apache.sshd:sshd-sftp:2.0.0' - implementation 'org.apache.tomcat:servlet-api:6.0.45' - implementation 'org.apache.tomcat:tomcat-annotations-api:8.0.5' - implementation 'org.bouncycastle:bcpg-jdk15on:1.52' - implementation 'org.bouncycastle:bcpkix-jdk15on:1.52' - implementation 'org.bouncycastle:bcprov-jdk15on:1.52' - implementation 'org.codehaus.jackson:jackson-core-asl:1.9.13' - implementation 'org.codehaus.jackson:jackson-mapper-asl:1.9.13' - implementation 'org.joda:joda-money:0.10.0' - implementation 'org.json:json:20160810' - implementation 'org.khronos:opengl-api:gl1.1-android-2.1_r1' - implementation 'org.mortbay.jetty:jetty:6.1.26' - implementation 'org.mortbay.jetty:servlet-api:2.5-20081211' - implementation 'org.mortbay.jetty:jetty-util:6.1.26' - implementation 'org.slf4j:slf4j-api:1.7.16' - implementation 'org.tukaani:xz:1.5' - implementation 'org.xerial.snappy:snappy-java:1.1.4-M3' - implementation 'org.yaml:snakeyaml:1.17' - implementation 'xerces:xmlParserAPIs:2.6.2' - implementation 'xpp3:xpp3:1.1.4c' - // Custom-built objectify jar at commit ecd5165, included in Nomulus release. - implementation name: 'objectify-4.1.3' + // Custom-built objectify jar at commit ecd5165, included in Nomulus + // release. + implementation files( + "${rootDir}/../third_party/objectify/v4_1/objectify-4.1.3.jar") + testImplementation project(':third_party') - compileOnly 'com.google.appengine:appengine-remote-api:1.9.48' // Also testImplementation - compileOnly 'com.google.auto.service:auto-service:1.0-rc4' - compileOnly 'org.osgi:org.osgi.core:4.3.0' + compile 'com.beust:jcommander:1.48' + compile 'com.google.api-client:google-api-client:1.22.0' + maybeRuntime 'com.google.api-client:google-api-client-appengine:1.22.0' + maybeRuntime 'com.google.api-client:google-api-client-jackson2:1.20.0' + compile 'com.google.monitoring-client:metrics:1.0.4' + compile 'com.google.monitoring-client:stackdriver:1.0.4' + compile 'com.google.api-client:google-api-client-java6:1.27.0' + maybeRuntime 'com.google.api-client:google-api-client-servlet:1.22.0' + compile 'com.google.apis:google-api-services-admin-directory:directory_v1-rev72-1.22.0' + compile 'com.google.apis:google-api-services-appengine:v1-rev85-1.25.0' + compile 'com.google.apis:google-api-services-bigquery:v2-rev325-1.22.0' + maybeRuntime 'com.google.apis:google-api-services-clouddebugger:v2-rev8-1.22.0' + compile 'com.google.apis:google-api-services-cloudkms:v1-rev12-1.22.0' + maybeRuntime 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev6-1.22.0' + compile 'com.google.apis:google-api-services-dataflow:v1b3-rev196-1.22.0' + compile 'com.google.apis:google-api-services-dns:v2beta1-rev6-1.22.0' + compile 'com.google.apis:google-api-services-drive:v2-rev160-1.19.1' + compile 'com.google.apis:google-api-services-groupssettings:v1-rev60-1.22.0' + compile 'com.google.apis:google-api-services-monitoring:v3-rev11-1.22.0' + compile 'com.google.apis:google-api-services-sheets:v4-rev483-1.22.0' + maybeRuntime 'com.google.apis:google-api-services-storage:v1-rev86-1.22.0' + // TODO(b/71631624): change appengine:appengine-api-1.0-sdk to + // testCompileOnly after BillingEmailUtilsTest.java is fixed. + compile 'com.google.appengine:appengine-api-1.0-sdk:1.9.48' + maybeRuntime 'com.google.appengine:appengine-api-labs:1.9.48' + maybeRuntime 'com.google.appengine:appengine-api-stubs:1.9.48' + testCompile 'com.google.appengine:appengine-api-stubs:1.9.48' + compile 'com.google.appengine.tools:appengine-gcs-client:0.6' + compile 'com.google.appengine.tools:appengine-mapreduce:0.8.5' + compile 'com.google.appengine.tools:appengine-pipeline:0.2.13' + compile 'com.google.appengine:appengine-remote-api:1.9.48' + maybeRuntime 'com.google.appengine:appengine-tools-sdk:1.9.48' + compile 'com.google.auth:google-auth-library-credentials:0.7.1' + compile 'com.google.auth:google-auth-library-oauth2-http:0.7.1' + maybeRuntime 'com.google.auto:auto-common:0.8' + maybeRuntime 'com.google.auto.factory:auto-factory:1.0-beta3' + compile 'com.google.code.gson:gson:2.8.5' + compile 'com.google.auto.value:auto-value-annotations:1.6.2' + maybeRuntime 'com.google.cloud.bigdataoss:gcsio:1.4.5' + maybeRuntime 'com.google.cloud.bigdataoss:util:1.4.5' + compile 'com.google.code.findbugs:jsr305:3.0.2' + compile 'com.google.dagger:dagger:2.15' + maybeRuntime 'com.google.dagger:dagger-producers:2.15' + compile 'com.google.errorprone:error_prone_annotations:2.3.1' + maybeRuntime 'com.google.errorprone:javac-shaded:9-dev-r4023-3' + compile 'com.google.flogger:flogger:0.1' + runtime 'com.google.flogger:flogger-system-backend:0.1' + maybeRuntime 'com.google.gdata:core:1.47.1' + maybeRuntime 'com.google.googlejavaformat:google-java-format:1.4' + compile 'com.google.guava:guava:25.1-jre' + gradleLint.ignore('unused-dependency') { + compile 'com.google.gwt:gwt-user:2.8.2' + } + compile 'com.google.http-client:google-http-client:1.25.0' + compile 'com.google.http-client:google-http-client-appengine:1.22.0' + compile 'com.google.http-client:google-http-client-jackson2:1.25.0' + compile 'com.google.oauth-client:google-oauth-client:1.25.0' + maybeRuntime 'com.google.oauth-client:google-oauth-client-appengine:1.22.0' + compile 'com.google.oauth-client:google-oauth-client-java6:1.27.0' + compile 'com.google.oauth-client:google-oauth-client-jetty:1.22.0' + maybeRuntime 'com.google.oauth-client:google-oauth-client-servlet:1.22.0' + maybeRuntime 'com.google.protobuf:protobuf-java:2.6.0' + compile 'com.google.re2j:re2j:1.1' + compile 'com.google.template:soy:2018-03-14' + maybeRuntime 'com.googlecode.charts4j:charts4j:1.3' + compile 'com.googlecode.json-simple:json-simple:1.1.1' + compile 'com.jcraft:jsch:0.1.53' + maybeRuntime 'com.jcraft:jzlib:1.1.3' + maybeRuntime 'com.squareup:javapoet:1.8.0' + maybeRuntime 'com.squareup:javawriter:2.5.1' + maybeRuntime 'com.sun.activation:javax.activation:1.2.0' + maybeRuntime 'com.thoughtworks.paranamer:paranamer:2.7' + maybeRuntime 'commons-codec:commons-codec:1.10' + compile group: 'commons-io', name: 'commons-io', version: '2.6' + maybeRuntime 'commons-logging:commons-logging:1.2' + compile 'dnsjava:dnsjava:2.1.7' + maybeRuntime 'io.netty:netty-buffer:4.1.28.Final' + maybeRuntime 'io.netty:netty-codec:4.1.28.Final' + maybeRuntime 'io.netty:netty-codec-http:4.1.28.Final' + maybeRuntime 'io.netty:netty-common:4.1.28.Final' + maybeRuntime 'io.netty:netty-handler:4.1.28.Final' + maybeRuntime 'io.netty:netty-resolver:4.1.28.Final' + maybeRuntime 'io.netty:netty-tcnative:2.0.12.Final' + maybeRuntime 'io.netty:netty-tcnative-boringssl-static:2.0.12.Final' + maybeRuntime 'io.netty:netty-transport:4.1.28.Final' + maybeRuntime 'it.unimi.dsi:fastutil:6.5.16' + maybeRuntime 'javax.annotation:jsr250-api:1.0' + runtime 'org.glassfish.jaxb:jaxb-runtime:2.3.0' + testCompile 'javax.annotation:jsr250-api:1.0' + compile 'javax.inject:javax.inject:1' + compile 'javax.mail:mail:1.4' + compile 'javax.servlet:servlet-api:2.5' + compile 'javax.xml.bind:jaxb-api:2.3.0' + maybeRuntime 'javax.xml.soap:javax.xml.soap-api:1.4.0' + compile 'jline:jline:1.0' + compile 'joda-time:joda-time:2.3' + compile 'org.apache.avro:avro:1.8.2' + maybeRuntime 'org.apache.beam:beam-runners-direct-java:2.2.0' + testCompile 'org.apache.beam:beam-runners-direct-java:2.2.0' + compile 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.1.0' + maybeRuntime 'org.apache.beam:beam-sdks-common-runner-api:2.1.0' + compile 'org.apache.beam:beam-sdks-java-core:2.2.0' + compile 'org.apache.beam:beam-sdks-java-extensions-google-cloud-platform-core:2.1.0' + compile 'org.apache.beam:beam-sdks-java-io-google-cloud-platform:2.2.0' + maybeRuntime 'org.apache.commons:commons-compress:1.8.1' + maybeRuntime 'org.apache.ftpserver:ftplet-api:1.0.6' + testCompile 'org.apache.ftpserver:ftplet-api:1.0.6' + maybeRuntime 'org.apache.ftpserver:ftpserver-core:1.0.6' + testCompile 'org.apache.ftpserver:ftpserver-core:1.0.6' + compile 'org.apache.httpcomponents:httpclient:4.5.2' + compile 'org.apache.httpcomponents:httpcore:4.4.4' + maybeRuntime 'org.apache.mina:mina-core:2.0.4' + maybeRuntime 'org.apache.sshd:sshd-core:2.0.0' + testCompile 'org.apache.sshd:sshd-core:2.0.0' + maybeRuntime 'org.apache.sshd:sshd-scp:2.0.0' + testCompile 'org.apache.sshd:sshd-scp:2.0.0' + maybeRuntime 'org.apache.sshd:sshd-sftp:2.0.0' + testCompile 'org.apache.sshd:sshd-sftp:2.0.0' + compile 'org.apache.tomcat:servlet-api:6.0.45' + maybeRuntime 'org.apache.tomcat:tomcat-annotations-api:8.0.5' + testCompile 'org.apache.tomcat:tomcat-annotations-api:8.0.5' + compile 'org.bouncycastle:bcpg-jdk15on:1.52' + testCompile 'org.bouncycastle:bcpkix-jdk15on:1.52' + compile 'org.bouncycastle:bcprov-jdk15on:1.52' + maybeRuntime 'org.codehaus.jackson:jackson-core-asl:1.9.13' + maybeRuntime 'org.codehaus.jackson:jackson-mapper-asl:1.9.13' + compile 'org.joda:joda-money:0.10.0' + compile 'org.json:json:20160810' + maybeRuntime 'org.khronos:opengl-api:gl1.1-android-2.1_r1' + maybeRuntime 'org.mortbay.jetty:jetty:6.1.26' + testCompile 'org.mortbay.jetty:jetty:6.1.26' + compile 'org.mortbay.jetty:servlet-api:2.5-20081211' + maybeRuntime 'org.mortbay.jetty:jetty-util:6.1.26' + maybeRuntime 'org.slf4j:slf4j-api:1.7.16' + maybeRuntime 'org.tukaani:xz:1.5' + maybeRuntime 'org.xerial.snappy:snappy-java:1.1.4-M3' + compile 'xerces:xmlParserAPIs:2.6.2' + compile 'xpp3:xpp3:1.1.4c' + // Known issue: nebula-lint misses inherited dependency. + compile project(':third_party') + compile project(':util') + + // Include auto-value in compile until nebula-lint understands + // annotationProcessor + gradleLint.ignore('unused-dependency') { + compile 'com.google.auto.value:auto-value:1.6.2' + } annotationProcessor 'com.google.auto.value:auto-value:1.6.2' testAnnotationProcessor 'com.google.auto.value:auto-value:1.6.2' annotationProcessor 'com.google.dagger:dagger-compiler:2.15' testAnnotationProcessor 'com.google.dagger:dagger-compiler:2.15' - testImplementation 'com.google.appengine:appengine-remote-api:1.9.48' // Also compileOnly - testImplementation 'com.google.appengine:appengine-testing:1.9.58' - testImplementation 'com.google.guava:guava-testlib:25.0-jre' - testImplementation 'com.google.monitoring-client:contrib:1.0.4' - testImplementation 'com.google.truth:truth:0.42' - testImplementation 'com.google.truth.extensions:truth-java8-extension:0.39' - testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation 'org.hamcrest:hamcrest-core:1.3' - testImplementation 'org.hamcrest:hamcrest-library:1.3' - testImplementation 'junit:junit:4.12' - testImplementation 'org.mockito:mockito-all:1.9.5' - testImplementation 'org.slf4j:slf4j-simple:1.7.16' // Not needed by Bazel + testCompile 'com.google.appengine:appengine-testing:1.9.58' + testCompile 'com.google.guava:guava-testlib:25.0-jre' + testCompile 'com.google.monitoring-client:contrib:1.0.4' + testCompile 'com.google.truth:truth:0.42' + testCompile 'com.google.truth.extensions:truth-java8-extension:0.39' + testCompile 'org.hamcrest:hamcrest-all:1.3' + testCompile 'org.hamcrest:hamcrest-core:1.3' + testCompile 'org.hamcrest:hamcrest-library:1.3' + testCompile 'junit:junit:4.12' + testCompile 'org.mockito:mockito-all:1.9.5' - testImplementation project(':third_party') + // Indirect dependency found by undeclared-dependency check. Such + // dependencies should go after all other compile and testCompile + // dependencies to avoid overriding them accidentally. + compile 'javax.servlet:javax.servlet-api:3.1.0' // google-api-client-appeng + compile 'com.google.oauth-client:google-oauth-client-java6:1.20.0' // Dependencies needed for jaxb compilation. // Use jaxb 2.2.11 because 2.3 is known to break the Ant task we use. @@ -217,14 +263,18 @@ dependencies { // Dependencies needed for compiling stylesheets to javascript css 'com.google.closure-stylesheets:closure-stylesheets:1.5.0' css 'args4j:args4j:2.0.26' + + // Tool dependencies. used for doc generation. + compile files("${System.properties['java.home']}/../lib/tools.jar") } -task jaxbToJava() { +task jaxbToJava { doLast { file(generatedDir).mkdirs() - // Temp dir to hold schema and bindings files. Files must be in the same directory because - // the bindings (.xjb) file does not declare relative paths to schema (.xsd) files. + // Temp dir to hold schema and bindings files. Files must be in the same + // directory because the bindings (.xjb) file does not declare relative + // paths to schema (.xsd) files. def xjcTempSourceDir = file("${temporaryDir}/xjc") xjcTempSourceDir.mkdirs() ant.copy( @@ -252,7 +302,8 @@ task jaxbToJava() { dir: new File("$xjcTempSourceDir"), include: ['**/*.xsd']) .addToAntBuilder(ant, 'schema', FileCollection.AntType.FileSet) - // -npa: do not generate package-info.java files. They will be generated below. + // -npa: do not generate package-info.java files. They will be generated + // below. arg(line: '-npa -quiet -extension') } exec { @@ -265,7 +316,7 @@ task jaxbToJava() { } } -task soyToJava() { +task soyToJava { ext.soyToJava = { javaPackage, outputDirectory, soyFiles -> javaexec { main = "com.google.template.soy.SoyParseInfoGenerator" @@ -280,12 +331,23 @@ task soyToJava() { doLast { - soyToJava('google.registry.tools.soy', "${generatedDir}/google/registry/tools/soy", - fileTree(dir: "${javaDir}/google/registry/tools/soy", include: ['**/*.soy'])) + soyToJava('google.registry.tools.soy', + "${generatedDir}/google/registry/tools/soy", + fileTree( + dir: "${javaDir}/google/registry/tools/soy", + include: ['**/*.soy'])) soyToJava('google.registry.ui.soy.registrar', "${generatedDir}/google/registry/ui/soy/registrar", - fileTree(dir: "${javaDir}/google/registry/ui/soy/registrar", include: ['**/*.soy'])) + fileTree( + dir: "${javaDir}/google/registry/ui/soy/registrar", + include: ['**/*.soy'])) + + soyToJava('google.registry.ui.soy.otesetup', + "${generatedDir}/google/registry/ui/soy/otesetup", + fileTree( + dir: "${javaDir}/google/registry/ui/soy/otesetup", + include: ['**/*.soy'])) soyToJava('google.registry.ui.soy', "${generatedDir}/google/registry/ui/soy", @@ -294,6 +356,14 @@ task soyToJava() { }.filter { it.name.endsWith(".soy") }) + + soyToJava('google.registry.ui.soy.otesetup', + "${generatedDir}/google/registry/ui/soy/otesetup", + files { + file("${javaDir}/google/registry/ui/soy/otesetup").listFiles() + }.filter { + it.name.endsWith(".soy") + }) } } @@ -327,11 +397,16 @@ task stylesheetsToJavascript { def outputDir = "${project.buildDir}/resources/main/google/registry/ui/css" file("${outputDir}").mkdirs() def srcFiles = [ - "${cssSourceDir}/console.css", "${cssSourceDir}/contact-settings.css", - "${cssSourceDir}/contact-us.css", "${cssSourceDir}/dashboard.css", - "${cssSourceDir}/epp.css", "${cssSourceDir}/forms.css", - "${cssSourceDir}/kd_components.css", "${cssSourceDir}/registry.css", - "${cssSourceDir}/resources.css", "${cssSourceDir}/security-settings.css" + "${cssSourceDir}/console.css", + "${cssSourceDir}/contact-settings.css", + "${cssSourceDir}/contact-us.css", + "${cssSourceDir}/dashboard.css", + "${cssSourceDir}/epp.css", + "${cssSourceDir}/forms.css", + "${cssSourceDir}/kd_components.css", + "${cssSourceDir}/registry.css", + "${cssSourceDir}/resources.css", + "${cssSourceDir}/security-settings.css" ] cssCompile("${outputDir}/registrar_bin", false, srcFiles) cssCompile("${outputDir}/registrar_dbg", true, srcFiles) @@ -341,26 +416,72 @@ task stylesheetsToJavascript { compileJava.dependsOn jaxbToJava compileJava.dependsOn soyToJava -// stylesheetsToJavascript must happen after processResources, which wipes the resources folder -// before copying data into it. +// stylesheetsToJavascript must happen after processResources, which wipes the +// resources folder before copying data into it. stylesheetsToJavascript.dependsOn processResources classes.dependsOn stylesheetsToJavascript +// Make testing artifacts available to be depended up on by other projects. +// TODO: factor out google.registry.testing to be a separate project. +task testJar(type: Jar) { + classifier = 'test' + from sourceSets.test.output +} -test { - // Test exclusion patterns: - // - *TestCase.java are inherited by concrete test classes. - // - *TestSuite.java are excluded to avoid duplicate execution of suite members. See README - // in this directory for more information. +artifacts { + testRuntime testJar +} + +task fragileTest(type: Test) { + // Common exclude pattern. See README in parent directory for explanation. exclude "**/*TestCase.*", "**/*TestSuite.*" + include fragileTestPatterns - // Use a single JVM to execute all tests. See README in this directory for more information. - maxParallelForks 1 - - // Use a single thread to execute all tests in a JVM. See README in this directory for more - // information. + // Run every test class in a freshly started process. forkEvery 1 // Uncomment to see test outputs in stdout. //testLogging.showStandardStreams = true } + +task outcastTest(type: Test) { + // Common exclude pattern. See README in parent directory for explanation. + exclude "**/*TestCase.*", "**/*TestSuite.*" + include outcastTestPatterns + + // Sets the maximum number of test executors that may exist at the same time. + maxParallelForks 5 +} + +test { + // Common exclude pattern. See README in parent directory for explanation. + exclude "**/*TestCase.*", "**/*TestSuite.*" + exclude fragileTestPatterns + exclude outcastTestPatterns + + // Sets the maximum number of test executors that may exist at the same time. + maxParallelForks 5 +}.dependsOn(fragileTest, outcastTest) + +task nomulus(type: Jar) { + manifest { + attributes 'Main-Class': 'google.registry.tools.RegistryTool' + } + zip64 = true + baseName = 'nomulus' + version = null + from { + configurations.runtimeClasspath.collect { + it.isDirectory() ? it : zipTree(it) + } + } + // Excludes signature files that accompany some dependency jars, like + // bonuncycastle. It they are present, only classes from those signed jars are + // made available to the class loader. + // see https://discuss.gradle.org/t/signing-a-custom-gradle-plugin-thats-downloaded-by-the-build-system-from-github/1365 + exclude "META-INF/*.SF", "META-INF/*.DSA", "META-INF/*.RSA" + with jar + dependsOn project(':third_party').jar +} + +ext.generateDependencyPublications() diff --git a/gradle/generate_dependency_metadata.sh b/gradle/generate_dependency_metadata.sh new file mode 100755 index 000000000..394b63271 --- /dev/null +++ b/gradle/generate_dependency_metadata.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2018 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script runs a workflow to clone a git repository to local, generate +# a metadata file for each dependency artifact and check in the file to remote +# repository. + +set -e + +ALL_SUBPROJECTS="core proxy util" + +USAGE="Usage: ${0} REPO_URL" +REPO_URL=${1:?${USAGE}} + +REPO_DIR="$(mktemp -d)" + +git clone ${REPO_URL} ${REPO_DIR} +for PROJECT in ${ALL_SUBPROJECTS}; do + $(dirname $0)/gradlew -PprivateRepository="${REPO_DIR}" \ + ":${PROJECT}:generateDependencyMetadata" +done +cd "${REPO_DIR}" +git add -A +git diff-index --quiet HEAD \ + || git commit -m "Update dependency metadata file" && git push +rm -rf "${REPO_DIR}" diff --git a/gradle/gradle/wrapper/gradle-wrapper.jar b/gradle/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..457aad0d9 Binary files /dev/null and b/gradle/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/gradle/wrapper/gradle-wrapper.properties b/gradle/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..75b8c7c8c --- /dev/null +++ b/gradle/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradle/gradlew b/gradle/gradlew new file mode 100644 index 000000000..af6708ff2 --- /dev/null +++ b/gradle/gradlew @@ -0,0 +1,172 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/gradle/gradlew.bat b/gradle/gradlew.bat new file mode 100644 index 000000000..6d57edc70 --- /dev/null +++ b/gradle/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/gradle/proxy/Dockerfile b/gradle/proxy/Dockerfile new file mode 100644 index 000000000..bad276559 --- /dev/null +++ b/gradle/proxy/Dockerfile @@ -0,0 +1,5 @@ +# TODO(jianglai): Peg to a specific sha256 hash to enable reproducible build. +FROM gcr.io/distroless/java +ADD build/libs/proxy_server.jar . +ENTRYPOINT ["java", "-jar", "proxy_server.jar"] +EXPOSE 30000 30001 30002 30010 30012 diff --git a/gradle/proxy/build.gradle b/gradle/proxy/build.gradle new file mode 100644 index 000000000..2dc363e33 --- /dev/null +++ b/gradle/proxy/build.gradle @@ -0,0 +1,103 @@ +apply plugin: 'com.google.osdetector' +apply plugin: 'application' +apply plugin: 'com.bmuschko.docker-java-application' + +// TODO(jianglai): use plugins block once the osdetctor v1.6.0 works with it. +// see: https://github.com/google/osdetector-gradle-plugin/issues/15 +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath 'com.google.gradle:osdetector-gradle-plugin:1.6.0' + } +} + +sourceSets { + main { + resources { + exclude "${project.relativePath}/terraform/" + exclude "${project.relativePath}/kubernetes/" + } + } +} + +mainClassName = 'google.registry.proxy.ProxyServer' + +task deployJar(type: Jar) { + manifest { + attributes 'Main-Class': 'google.registry.proxy.ProxyServer' + } + baseName = 'proxy_server' + version = null + from { + configurations.runtimeClasspath.collect { + it.isDirectory() ? it : zipTree(it) + } + } + // Excludes signature files that accompany some dependency jars, like + // bonuncycastle. It they are present, only classes from those signed jars are + // made available to the class loader. + // see https://discuss.gradle.org/t/signing-a-custom-gradle-plugin-thats-downloaded-by-the-build-system-from-github/1365 + exclude "META-INF/*.SF", "META-INF/*.DSA", "META-INF/*.RSA" + with jar +} + +dependencies { + compile 'com.beust:jcommander:1.48' + compile 'com.google.api-client:google-api-client:1.27.0' + compile 'com.google.api-client:google-api-client:1.27.0' + compile 'com.google.apis:google-api-services-cloudkms:v1-rev12-1.22.0' + compile 'com.google.apis:google-api-services-monitoring:v3-rev11-1.22.0' + compile 'com.google.apis:google-api-services-storage:v1-rev86-1.22.0' + compile 'com.google.auto.value:auto-value-annotations:1.6.2' + compile 'com.google.code.findbugs:jsr305:3.0.2' + compile 'com.google.code.gson:gson:2.8.5' + compile 'com.google.dagger:dagger:2.15' + compile 'com.google.flogger:flogger:0.1' + compile 'com.google.guava:guava:27.0-jre' + compile 'com.google.http-client:google-http-client:1.27.0' + compile 'com.google.monitoring-client:metrics:1.0.4' + compile 'com.google.monitoring-client:stackdriver:1.0.4' + compile 'io.netty:netty-buffer:4.1.31.Final' + compile 'io.netty:netty-codec-http:4.1.31.Final' + compile 'io.netty:netty-codec:4.1.31.Final' + compile 'io.netty:netty-common:4.1.31.Final' + compile 'io.netty:netty-handler:4.1.31.Final' + compile 'io.netty:netty-transport:4.1.31.Final' + compile 'javax.inject:javax.inject:1' + compile 'joda-time:joda-time:2.3' + compile 'org.bouncycastle:bcpkix-jdk15on:1.52' + compile 'org.bouncycastle:bcprov-jdk15on:1.52' + compile project(':util') + + runtime 'com.google.flogger:flogger-system-backend:0.1' + runtime 'com.google.auto.value:auto-value:1.6.2' + runtime group: 'io.netty', name: 'netty-tcnative-boringssl-static', + version: '2.0.20.Final', classifier: osdetector.classifier + + testCompile 'com.google.monitoring-client:contrib:1.0.4' + testCompile 'com.google.truth:truth:0.42' + testCompile 'org.yaml:snakeyaml:1.17' + testCompile 'junit:junit:4.12' + testCompile 'org.mockito:mockito-all:1.9.5' + testCompile project(':third_party') + testCompile project(path: ':core', configuration: 'testRuntime') + + // Include auto-value in compile until nebula-lint understands + // annotationProcessor + annotationProcessor 'com.google.auto.value:auto-value:1.6.2' + testAnnotationProcessor 'com.google.auto.value:auto-value:1.6.2' + annotationProcessor 'com.google.dagger:dagger-compiler:2.15' + testAnnotationProcessor 'com.google.dagger:dagger-compiler:2.15' +} + +docker { + javaApplication { + // TODO(jianglai): Peg to a specific hash to enable reproducible build. + baseImage = 'openjdk:8-jre-alpine' + ports = [30000, 30001, 30002, 30011, 30012] + } +} + +ext.generateDependencyPublications() diff --git a/gradle/services/backend/README.txt b/gradle/services/backend/README.txt new file mode 100644 index 000000000..956793fc4 --- /dev/null +++ b/gradle/services/backend/README.txt @@ -0,0 +1,2 @@ +This directory is intentionally empty. This subproject is configred through root +project build script. diff --git a/gradle/services/default/README.txt b/gradle/services/default/README.txt new file mode 100644 index 000000000..956793fc4 --- /dev/null +++ b/gradle/services/default/README.txt @@ -0,0 +1,2 @@ +This directory is intentionally empty. This subproject is configred through root +project build script. diff --git a/gradle/services/pubapi/README.txt b/gradle/services/pubapi/README.txt new file mode 100644 index 000000000..956793fc4 --- /dev/null +++ b/gradle/services/pubapi/README.txt @@ -0,0 +1,2 @@ +This directory is intentionally empty. This subproject is configred through root +project build script. diff --git a/gradle/services/tools/README.txt b/gradle/services/tools/README.txt new file mode 100644 index 000000000..956793fc4 --- /dev/null +++ b/gradle/services/tools/README.txt @@ -0,0 +1,2 @@ +This directory is intentionally empty. This subproject is configred through root +project build script. diff --git a/gradle/settings.gradle b/gradle/settings.gradle index 3a9c6015f..30879379f 100644 --- a/gradle/settings.gradle +++ b/gradle/settings.gradle @@ -1,5 +1,10 @@ rootProject.name = 'nomulus' -include 'third_party' include 'core' - +include 'proxy' +include 'third_party' +include 'util' +include 'services:default' +include 'services:backend' +include 'services:tools' +include 'services:pubapi' diff --git a/gradle/update_dependency.sh b/gradle/update_dependency.sh new file mode 100755 index 000000000..b7cf3f254 --- /dev/null +++ b/gradle/update_dependency.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2018 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script runs a workflow to generate dependency lock file, run a build against +# the generated lock file, save the lock file and upload dependency JARs to a private +# Maven repository if the build succeeds. + +set -e + +ALL_SUBPROJECTS="core proxy util" +SUBPROJECTS= +REPOSITORY_URL= + +while [[ $# -gt 0 ]]; do + KEY="$1" + case ${KEY} in + --repositoryUrl) + shift + REPOSITORY_URL="$1" + ;; + *) + SUBPROJECTS="${SUBPROJECTS} ${KEY}" + ;; + esac + shift +done + +if [[ -z ${SUBPROJECTS} ]]; then + SUBPROJECTS="${ALL_SUBPROJECTS}" +fi + +if [[ -z ${REPOSITORY_URL} ]]; then + echo "--repositoryUrl must be specified" + exit 1 +fi + +WORKING_DIR=$(dirname $0) + +for PROJECT in ${SUBPROJECTS}; do + ${WORKING_DIR}/gradlew ":${PROJECT}:generateLock" + ${WORKING_DIR}/gradlew -PdependencyLock.useGeneratedLock=true \ + ":${PROJECT}:build" + ${WORKING_DIR}/gradlew ":${PROJECT}:saveLock" + ${WORKING_DIR}/gradlew -PrepositoryUrl="${REPOSITORY_URL}" \ + ":${PROJECT}:publish" +done diff --git a/gradle/util/build.gradle b/gradle/util/build.gradle new file mode 100644 index 000000000..04dd7686e --- /dev/null +++ b/gradle/util/build.gradle @@ -0,0 +1,28 @@ +dependencies { + compile 'com.google.appengine:appengine-api-1.0-sdk:1.9.48' + compile 'com.google.appengine:appengine-testing:1.9.58' + compile 'com.google.code.findbugs:jsr305:3.0.2' + compile 'com.google.dagger:dagger:2.15' + compile 'com.google.flogger:flogger:0.1' + compile 'com.google.guava:guava:25.1-jre' + compile 'com.google.re2j:re2j:1.1' + compile 'com.ibm.icu:icu4j:57.1' + compile 'javax.inject:javax.inject:1' + compile 'javax.mail:mail:1.4' + compile 'javax.xml.bind:jaxb-api:2.3.0' + compile 'joda-time:joda-time:2.9.2' + compile 'nomulus:util:1.0' + compile 'org.yaml:snakeyaml:1.17' + testCompile 'com.google.appengine:appengine-api-stubs:1.9.48' + testCompile 'com.google.guava:guava-testlib:25.0-jre' + testCompile 'com.google.truth:truth:0.42' + testCompile 'junit:junit:4.12' + testCompile 'org.hamcrest:hamcrest-all:1.3' + testCompile 'org.hamcrest:hamcrest-core:1.3' + testCompile 'org.mockito:mockito-all:1.9.5' + testCompile files("${rootDir}/../third_party/objectify/v4_1/objectify-4.1.3.jar") + testCompile project(':third_party') + testCompile project(path: ':core', configuration: 'testRuntime') + annotationProcessor 'com.google.dagger:dagger-compiler:2.15' + testAnnotationProcessor 'com.google.dagger:dagger-compiler:2.15' +} diff --git a/java/google/registry/batch/DeleteContactsAndHostsAction.java b/java/google/registry/batch/DeleteContactsAndHostsAction.java index 2bbd1bb88..675221a9e 100644 --- a/java/google/registry/batch/DeleteContactsAndHostsAction.java +++ b/java/google/registry/batch/DeleteContactsAndHostsAction.java @@ -39,6 +39,7 @@ import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE; import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE_FAILURE; import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE; import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE_FAILURE; +import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED; import static google.registry.util.PipelineUtils.createJobPath; import static java.math.RoundingMode.CEILING; import static java.util.concurrent.TimeUnit.DAYS; @@ -85,7 +86,6 @@ import google.registry.model.poll.PendingActionNotificationResponse.HostPendingA import google.registry.model.poll.PollMessage; import google.registry.model.reporting.HistoryEntry; import google.registry.model.server.Lock; -import google.registry.model.transfer.TransferStatus; import google.registry.request.Action; import google.registry.request.Response; import google.registry.request.auth.Auth; @@ -394,7 +394,9 @@ public class DeleteContactsAndHostsAction implements Runnable { ContactResource contact = (ContactResource) resource; // Handle pending transfers on contact deletion. if (contact.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) { - contact = denyPendingTransfer(contact, TransferStatus.SERVER_CANCELLED, now); + contact = + denyPendingTransfer( + contact, SERVER_CANCELLED, now, deletionRequest.requestingClientId()); } // Wipe out PII on contact deletion. resourceToSaveBuilder = contact.asBuilder().wipeOut(); diff --git a/java/google/registry/beam/invoicing/BUILD b/java/google/registry/beam/invoicing/BUILD index af2e4b32b..d94f93f0c 100644 --- a/java/google/registry/beam/invoicing/BUILD +++ b/java/google/registry/beam/invoicing/BUILD @@ -15,6 +15,7 @@ java_library( "//java/google/registry/reporting/billing", "//java/google/registry/util", "@com_google_apis_google_api_services_bigquery", + "@com_google_auth_library_oauth2_http", "@com_google_auto_value", "@com_google_dagger", "@com_google_flogger", diff --git a/java/google/registry/beam/invoicing/InvoicingPipeline.java b/java/google/registry/beam/invoicing/InvoicingPipeline.java index 6e3bfe632..f439b5290 100644 --- a/java/google/registry/beam/invoicing/InvoicingPipeline.java +++ b/java/google/registry/beam/invoicing/InvoicingPipeline.java @@ -14,11 +14,17 @@ package google.registry.beam.invoicing; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.auth.oauth2.GoogleCredentials; import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey; import google.registry.beam.invoicing.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder; +import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.Config; import google.registry.reporting.billing.BillingModule; import google.registry.reporting.billing.GenerateInvoicesAction; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.Serializable; import javax.inject.Inject; import org.apache.beam.runners.dataflow.DataflowRunner; @@ -75,6 +81,12 @@ public class InvoicingPipeline implements Serializable { @Config("billingBucketUrl") String billingBucketUrl; + @Inject + @Config("invoiceFilePrefix") + String invoiceFilePrefix; + + @Inject @LocalCredentialJson String credentialJson; + @Inject InvoicingPipeline() {} @@ -96,6 +108,13 @@ public class InvoicingPipeline implements Serializable { public void deploy() { // We can't store options as a member variable due to serialization concerns. InvoicingPipelineOptions options = PipelineOptionsFactory.as(InvoicingPipelineOptions.class); + try { + options.setGcpCredential( + GoogleCredentials.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8)))); + } catch (IOException e) { + throw new RuntimeException( + "Cannot obtain local credential to deploy the invoicing pipeline", e); + } options.setProject(projectId); options.setRunner(DataflowRunner.class); // This causes p.run() to stage the pipeline as a template on GCS, as opposed to running it. @@ -164,7 +183,7 @@ public class InvoicingPipeline implements Serializable { billingBucketUrl, BillingModule.INVOICES_DIRECTORY, yearMonth, - BillingModule.OVERALL_INVOICE_PREFIX, + invoiceFilePrefix, yearMonth))) .withHeader(InvoiceGroupingKey.invoiceHeader()) .withoutSharding() diff --git a/java/google/registry/beam/spec11/BUILD b/java/google/registry/beam/spec11/BUILD index eefeb4c59..74696060c 100644 --- a/java/google/registry/beam/spec11/BUILD +++ b/java/google/registry/beam/spec11/BUILD @@ -12,6 +12,7 @@ java_library( "//java/google/registry/beam", "//java/google/registry/config", "//java/google/registry/util", + "@com_google_auth_library_oauth2_http", "@com_google_auto_value", "@com_google_dagger", "@com_google_flogger", diff --git a/java/google/registry/beam/spec11/Spec11Pipeline.java b/java/google/registry/beam/spec11/Spec11Pipeline.java index 3b80b0ae6..ee39720e6 100644 --- a/java/google/registry/beam/spec11/Spec11Pipeline.java +++ b/java/google/registry/beam/spec11/Spec11Pipeline.java @@ -39,6 +39,8 @@ import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.TypeDescriptor; import org.apache.beam.sdk.values.TypeDescriptors; +import org.joda.time.LocalDate; +import org.joda.time.YearMonth; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -56,13 +58,14 @@ import org.json.JSONObject; public class Spec11Pipeline implements Serializable { /** - * Returns the subdirectory spec11 reports reside in for a given yearMonth in yyyy-MM format. + * Returns the subdirectory spec11 reports reside in for a given local date in yyyy-MM-dd format. * * @see google.registry.beam.spec11.Spec11Pipeline * @see google.registry.reporting.spec11.Spec11EmailUtils */ - public static String getSpec11Subdirectory(String yearMonth) { - return String.format("icann/spec11/%s/SPEC11_MONTHLY_REPORT", yearMonth); + public static String getSpec11ReportFilePath(LocalDate localDate) { + YearMonth yearMonth = new YearMonth(localDate); + return String.format("icann/spec11/%s/SPEC11_MONTHLY_REPORT_%s", yearMonth, localDate); } /** The JSON object field we put the registrar's e-mail address for Spec11 reports. */ @@ -86,25 +89,24 @@ public class Spec11Pipeline implements Serializable { @Config("reportingBucketUrl") String reportingBucketUrl; - @Inject - Retrier retrier; + @Inject Retrier retrier; @Inject Spec11Pipeline() {} /** Custom options for running the spec11 pipeline. */ interface Spec11PipelineOptions extends DataflowPipelineOptions { - /** Returns the yearMonth we're generating the report for, in yyyy-MM format. */ - @Description("The yearMonth we generate the report for, in yyyy-MM format.") - ValueProvider getYearMonth(); + /** Returns the local date we're generating the report for, in yyyy-MM-dd format. */ + @Description("The local date we generate the report for, in yyyy-MM-dd format.") + ValueProvider getDate(); /** - * Sets the yearMonth we generate invoices for. + * Sets the local date we generate invoices for. * - *

This is implicitly set when executing the Dataflow template, by specifying the "yearMonth" + *

This is implicitly set when executing the Dataflow template, by specifying the "date" * parameter. */ - void setYearMonth(ValueProvider value); + void setDate(ValueProvider value); /** Returns the SafeBrowsing API key we use to evaluate subdomain health. */ @Description("The API key we use to access the SafeBrowsing API.") @@ -149,7 +151,7 @@ public class Spec11Pipeline implements Serializable { evaluateUrlHealth( domains, new EvaluateSafeBrowsingFn(options.getSafeBrowsingApiKey(), retrier), - options.getYearMonth()); + options.getDate()); p.run(); } @@ -161,7 +163,7 @@ public class Spec11Pipeline implements Serializable { void evaluateUrlHealth( PCollection domains, EvaluateSafeBrowsingFn evaluateSafeBrowsingFn, - ValueProvider yearMonthProvider) { + ValueProvider dateProvider) { domains .apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn)) .apply( @@ -199,13 +201,13 @@ public class Spec11Pipeline implements Serializable { TextIO.write() .to( NestedValueProvider.of( - yearMonthProvider, - yearMonth -> + dateProvider, + date -> String.format( "%s/%s", - reportingBucketUrl, getSpec11Subdirectory(yearMonth)))) + reportingBucketUrl, + getSpec11ReportFilePath(LocalDate.parse(date))))) .withoutSharding() .withHeader("Map from registrar email to detected subdomain threats:")); } - } diff --git a/java/google/registry/bigquery/BigqueryConnection.java b/java/google/registry/bigquery/BigqueryConnection.java index 2d2bf765a..7d7ac381b 100644 --- a/java/google/registry/bigquery/BigqueryConnection.java +++ b/java/google/registry/bigquery/BigqueryConnection.java @@ -676,7 +676,7 @@ public class BigqueryConnection implements AutoCloseable { bigquery.datasets().get(getProjectId(), datasetName).execute(); return true; } catch (GoogleJsonResponseException e) { - if (e.getDetails().getCode() == 404) { + if (e.getDetails() != null && e.getDetails().getCode() == 404) { return false; } throw e; @@ -689,7 +689,7 @@ public class BigqueryConnection implements AutoCloseable { bigquery.tables().get(getProjectId(), datasetName, tableName).execute(); return true; } catch (GoogleJsonResponseException e) { - if (e.getDetails().getCode() == 404) { + if (e.getDetails() != null && e.getDetails().getCode() == 404) { return false; } throw e; diff --git a/java/google/registry/bigquery/BigqueryJobFailureException.java b/java/google/registry/bigquery/BigqueryJobFailureException.java index 503238d61..99a68285d 100644 --- a/java/google/registry/bigquery/BigqueryJobFailureException.java +++ b/java/google/registry/bigquery/BigqueryJobFailureException.java @@ -30,15 +30,20 @@ public final class BigqueryJobFailureException extends RuntimeException { /** Delegate {@link IOException} errors, checking for {@link GoogleJsonResponseException} */ public static BigqueryJobFailureException create(IOException cause) { if (cause instanceof GoogleJsonResponseException) { - return create(((GoogleJsonResponseException) cause).getDetails()); + return create((GoogleJsonResponseException) cause); } else { return new BigqueryJobFailureException(cause.getMessage(), cause, null, null); } } /** Create an error for JSON server response errors. */ - public static BigqueryJobFailureException create(GoogleJsonError error) { - return new BigqueryJobFailureException(error.getMessage(), null, null, error); + public static BigqueryJobFailureException create(GoogleJsonResponseException cause) { + GoogleJsonError err = cause.getDetails(); + if (err != null) { + return new BigqueryJobFailureException(err.getMessage(), null, null, err); + } else { + return new BigqueryJobFailureException(cause.getMessage(), cause, null, null); + } } /** Create an error from a failed job. */ diff --git a/java/google/registry/builddefs/zip_file.bzl b/java/google/registry/builddefs/zip_file.bzl index a37456bc5..07c42691e 100644 --- a/java/google/registry/builddefs/zip_file.bzl +++ b/java/google/registry/builddefs/zip_file.bzl @@ -161,7 +161,7 @@ def _zip_file(ctx): for _, zip_path in mapped if "/" in zip_path ], - ) + ).to_list() ] cmd += [ 'ln -sf "${repo}/%s" "${tmp}/%s"' % (path, zip_path) @@ -181,7 +181,7 @@ def _zip_file(ctx): ctx.file_action(output = script, content = "\n".join(cmd), executable = True) inputs = [ctx.file._zipper] inputs += [dep.zip_file for dep in ctx.attr.deps] - inputs += list(srcs) + inputs += srcs.to_list() ctx.action( inputs = inputs, outputs = [ctx.outputs.out], diff --git a/java/google/registry/config/BUILD b/java/google/registry/config/BUILD index 812b0a13a..c0c22deac 100644 --- a/java/google/registry/config/BUILD +++ b/java/google/registry/config/BUILD @@ -12,6 +12,7 @@ java_library( "//java/google/registry/keyring/api", "//java/google/registry/util", "@com_google_api_client", + "@com_google_api_client_appengine", "@com_google_appengine_api_1_0_sdk", "@com_google_auto_value", "@com_google_code_findbugs_jsr305", @@ -23,6 +24,5 @@ java_library( "@javax_inject", "@joda_time", "@org_joda_money", - "@org_yaml_snakeyaml", ], ) diff --git a/java/google/registry/config/CredentialModule.java b/java/google/registry/config/CredentialModule.java index 446280be3..fba114fa4 100644 --- a/java/google/registry/config/CredentialModule.java +++ b/java/google/registry/config/CredentialModule.java @@ -26,6 +26,9 @@ import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; import java.security.GeneralSecurityException; import javax.inject.Qualifier; import javax.inject.Singleton; @@ -37,7 +40,21 @@ import javax.inject.Singleton; @Module public abstract class CredentialModule { - /** Provides the default {@link GoogleCredential} from the Google Cloud runtime. */ + /** + * Provides the default {@link GoogleCredential} from the Google Cloud runtime. + * + *

The credential returned depends on the runtime environment: + * + *

+ */ @DefaultCredential @Provides @Singleton @@ -109,6 +126,8 @@ public abstract class CredentialModule { /** Dagger qualifier for the Application Default Credential. */ @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) public @interface DefaultCredential {} /** @@ -116,6 +135,8 @@ public abstract class CredentialModule { * threads. */ @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) public @interface JsonCredential {} /** @@ -123,5 +144,19 @@ public abstract class CredentialModule { * Suite). */ @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) public @interface DelegatedCredential {} + + /** Dagger qualifier for the local credential used in the nomulus tool. */ + @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) + public @interface LocalCredential {} + + /** Dagger qualifier for the JSON string used to create the local credential. */ + @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) + public @interface LocalCredentialJson {} } diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 65f9883d5..9cb2f77f2 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -16,10 +16,13 @@ package google.registry.config; import static com.google.common.base.Suppliers.memoize; import static google.registry.config.ConfigUtils.makeUrl; +import static google.registry.util.ResourceUtils.readResourceUtf8; import static java.lang.annotation.RetentionPolicy.RUNTIME; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; import com.google.common.base.Splitter; +import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -29,6 +32,7 @@ import dagger.Provides; import google.registry.util.RandomStringGenerator; import google.registry.util.StringGenerator; import google.registry.util.TaskQueueUtils; +import google.registry.util.YamlUtils; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.net.URI; @@ -56,6 +60,10 @@ import org.joda.time.Duration; */ public final class RegistryConfig { + private static final String ENVIRONMENT_CONFIG_FORMAT = "files/nomulus-config-%s.yaml"; + private static final String YAML_CONFIG_PROD = + readResourceUtf8(RegistryConfig.class, "files/default-config.yaml"); + /** Dagger qualifier for configuration settings. */ @Qualifier @Retention(RUNTIME) @@ -64,6 +72,22 @@ public final class RegistryConfig { String value() default ""; } + /** + * Loads the {@link RegistryConfigSettings} POJO from the YAML configuration files. + * + *

The {@code default-config.yaml} file in this directory is loaded first, and a fatal error is + * thrown if it cannot be found or if there is an error parsing it. Separately, the + * environment-specific config file named {@code nomulus-config-ENVIRONMENT.yaml} is also loaded + * and those values merged into the POJO. + */ + static RegistryConfigSettings getConfigSettings() { + String configFilePath = + String.format( + ENVIRONMENT_CONFIG_FORMAT, Ascii.toLowerCase(RegistryEnvironment.get().name())); + String customYaml = readResourceUtf8(RegistryConfig.class, configFilePath); + return YamlUtils.getConfigSettings(YAML_CONFIG_PROD, customYaml, RegistryConfigSettings.class); + } + /** Dagger module for providing configuration settings. */ @Module public static final class ConfigModule { @@ -392,6 +416,20 @@ public final class RegistryConfig { return config.gSuite.adminAccountEmailAddress; } + /** + * Returns the email address of the group containing emails of support accounts. + * + *

These accounts will have "ADMIN" access to the registrar console. + * + * @see google.registry.groups.DirectoryGroupsConnection + */ + @Provides + @Config("gSuiteSupportGroupEmailAddress") + public static Optional provideGSuiteSupportGroupEmailAddress( + RegistryConfigSettings config) { + return Optional.ofNullable(Strings.emptyToNull(config.gSuite.supportGroupEmailAddress)); + } + /** * Returns the email address(es) that notifications of registrar and/or registrar contact * updates should be sent to, or the empty list if updates should not be sent. @@ -471,7 +509,7 @@ public final class RegistryConfig { /** * The email address that outgoing emails from the app are sent from. * - * @see google.registry.ui.server.registrar.SendEmailUtils + * @see google.registry.ui.server.SendEmailUtils */ @Provides @Config("gSuiteOutgoingEmailAddress") @@ -482,10 +520,10 @@ public final class RegistryConfig { /** * The display name that is used on outgoing emails sent by Nomulus. * - * @see google.registry.ui.server.registrar.SendEmailUtils + * @see google.registry.ui.server.SendEmailUtils */ @Provides - @Config("gSuiteOutoingEmailDisplayName") + @Config("gSuiteOutgoingEmailDisplayName") public static String provideGSuiteOutgoingEmailDisplayName(RegistryConfigSettings config) { return config.gSuite.outgoingEmailDisplayName; } @@ -537,6 +575,17 @@ public final class RegistryConfig { return beamBucketUrl + "/templates/spec11"; } + /** + * Returns whether an SSL certificate hash is required to log in via EPP and run flows. + * + * @see google.registry.flows.TlsCredentials + */ + @Provides + @Config("requireSslCertificates") + public static boolean provideRequireSslCertificates(RegistryConfigSettings config) { + return config.registryPolicy.requireSslCertificates; + } + /** * Returns the default job zone to run Apache Beam (Cloud Dataflow) jobs in. * @@ -656,6 +705,18 @@ public final class RegistryConfig { return ImmutableList.copyOf(config.billing.invoiceEmailRecipients); } + /** + * Returns the file prefix for the invoice CSV file. + * + * @see google.registry.beam.invoicing.InvoicingPipeline + * @see google.registry.reporting.billing.BillingEmailUtils + */ + @Provides + @Config("invoiceFilePrefix") + public static String provideInvoiceFilePrefix(RegistryConfigSettings config) { + return config.billing.invoiceFilePrefix; + } + /** * Returns the Google Cloud Storage bucket for staging escrow deposits pending upload. * @@ -827,20 +888,6 @@ public final class RegistryConfig { return config.misc.alertRecipientEmailAddress; } - /** - * Returns the email address we send emails from. - * - * @see google.registry.reporting.icann.ReportingEmailUtils - * @see google.registry.reporting.billing.BillingEmailUtils - * @see google.registry.reporting.spec11.Spec11EmailUtils - */ - @Provides - @Config("alertSenderEmailAddress") - public static String provideAlertSenderEmailAddress( - @Config("projectId") String projectId, RegistryConfigSettings config) { - return String.format("%s-no-reply@%s", projectId, config.misc.alertEmailSenderDomain); - } - /** * Returns the email address to which spec 11 email should be replied. * @@ -1090,6 +1137,12 @@ public final class RegistryConfig { return config.registryPolicy.allocationTokenCustomLogicClass; } + @Provides + @Config("dnsCountQueryCoordinatorClass") + public static String dnsCountQueryCoordinatorClass(RegistryConfigSettings config) { + return config.registryPolicy.dnsCountQueryCoordinatorClass; + } + /** Returns the disclaimer text for the exported premium terms. */ @Provides @Config("premiumTermsExportDisclaimer") @@ -1195,6 +1248,14 @@ public final class RegistryConfig { return ImmutableList.copyOf(config.credentialOAuth.delegatedCredentialOauthScopes); } + /** Provides the OAuth scopes required for credentials created locally for the nomulus tool. */ + @Provides + @Config("localCredentialOauthScopes") + public static ImmutableList provideLocalCredentialOauthScopes( + RegistryConfigSettings config) { + return ImmutableList.copyOf(config.credentialOAuth.localCredentialOauthScopes); + } + /** * Returns the help path for the RDAP terms of service. * @@ -1208,15 +1269,18 @@ public final class RegistryConfig { return "/tos"; } - /** - * Returns the name of the OAuth2 client secrets file. - * - *

This is the name of a resource relative to the root of the class tree. - */ + /** OAuth client ID used by the nomulus tool. */ @Provides - @Config("clientSecretFilename") - public static String provideClientSecretFilename(RegistryConfigSettings config) { - return config.registryTool.clientSecretFilename; + @Config("toolsClientId") + public static String provideToolsClientId(RegistryConfigSettings config) { + return config.registryTool.clientId; + } + + /** OAuth client secret used by the nomulus tool. */ + @Provides + @Config("toolsClientSecret") + public static String provideToolsClientSecret(RegistryConfigSettings config) { + return config.registryTool.clientSecret; } @Provides @@ -1225,6 +1289,18 @@ public final class RegistryConfig { return ImmutableList.copyOf(Splitter.on('\n').split(config.registryPolicy.rdapTos)); } + /** + * Link to static Web page with RDAP terms of service. Displayed in RDAP responses. + * + * @see google.registry.rdap.RdapJsonFormatter + */ + @Provides + @Config("rdapTosStaticUrl") + @Nullable + public static String provideRdapTosStaticUrl(RegistryConfigSettings config) { + return config.registryPolicy.rdapTosStaticUrl; + } + /** * Returns the help text to be used by RDAP. * @@ -1235,30 +1311,38 @@ public final class RegistryConfig { @Provides @Config("rdapHelpMap") public static ImmutableMap provideRdapHelpMap( - @Config("rdapTos") ImmutableList rdapTos) { + @Config("rdapTos") ImmutableList rdapTos, + @Config("rdapTosStaticUrl") @Nullable String rdapTosStaticUrl) { return new ImmutableMap.Builder() - .put("/", RdapNoticeDescriptor.builder() - .setTitle("RDAP Help") - .setDescription(ImmutableList.of( - "domain/XXXX", - "nameserver/XXXX", - "entity/XXXX", - "domains?name=XXXX", - "domains?nsLdhName=XXXX", - "domains?nsIp=XXXX", - "nameservers?name=XXXX", - "nameservers?ip=XXXX", - "entities?fn=XXXX", - "entities?handle=XXXX", - "help/XXXX")) - .setLinkValueSuffix("help/") - .setLinkHrefUrlString("https://github.com/google/nomulus/blob/master/docs/rdap.md") - .build()) - .put("/tos", RdapNoticeDescriptor.builder() - .setTitle("RDAP Terms of Service") - .setDescription(rdapTos) - .setLinkValueSuffix("help/tos") - .build()) + .put( + "/", + RdapNoticeDescriptor.builder() + .setTitle("RDAP Help") + .setDescription( + ImmutableList.of( + "domain/XXXX", + "nameserver/XXXX", + "entity/XXXX", + "domains?name=XXXX", + "domains?nsLdhName=XXXX", + "domains?nsIp=XXXX", + "nameservers?name=XXXX", + "nameservers?ip=XXXX", + "entities?fn=XXXX", + "entities?handle=XXXX", + "help/XXXX")) + .setLinkValueSuffix("help/") + .setLinkHrefUrlString( + "https://github.com/google/nomulus/blob/master/docs/rdap.md") + .build()) + .put( + "/tos", + RdapNoticeDescriptor.builder() + .setTitle("RDAP Terms of Service") + .setDescription(rdapTos) + .setLinkValueSuffix("help/tos") + .setLinkHrefUrlString(rdapTosStaticUrl) + .build()) .build(); } @@ -1319,6 +1403,15 @@ public final class RegistryConfig { return getProjectId() + "-snapshots"; } + /** + * Returns the Google Cloud Storage bucket for storing Datastore backups. + * + * @see google.registry.export.BackupDatastoreAction + */ + public static String getDatastoreBackupsBucket() { + return "gs://" + getProjectId() + "-datastore-backups"; + } + /** * Number of sharded commit log buckets. * @@ -1481,12 +1574,6 @@ public final class RegistryConfig { return Duration.standardDays(CONFIG_SETTINGS.get().registryPolicy.contactAutomaticTransferDays); } - /** Provided for testing. */ - @VisibleForTesting - public static String getClientSecretFilename() { - return CONFIG_SETTINGS.get().registryTool.clientSecretFilename; - } - /** * Memoizes loading of the {@link RegistryConfigSettings} POJO. * @@ -1495,7 +1582,7 @@ public final class RegistryConfig { */ @VisibleForTesting public static final Supplier CONFIG_SETTINGS = - memoize(YamlUtils::getConfigSettings); + memoize(RegistryConfig::getConfigSettings); private static String formatComments(String text) { return Splitter.on('\n').omitEmptyStrings().trimResults().splitToList(text).stream() diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index 58561f0fa..e7b1eac16 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -58,6 +58,7 @@ public class RegistryConfigSettings { public static class CredentialOAuth { public List defaultCredentialOauthScopes; public List delegatedCredentialOauthScopes; + public List localCredentialOauthScopes; } /** Configuration options for the G Suite account used by Nomulus. */ @@ -66,6 +67,7 @@ public class RegistryConfigSettings { public String outgoingEmailAddress; public String outgoingEmailDisplayName; public String adminAccountEmailAddress; + public String supportGroupEmailAddress; } /** Configuration options for registry policy. */ @@ -75,6 +77,7 @@ public class RegistryConfigSettings { public String customLogicFactoryClass; public String whoisCommandFactoryClass; public String allocationTokenCustomLogicClass; + public String dnsCountQueryCoordinatorClass; public int contactAutomaticTransferDays; public String greetingServerId; public List registrarChangesNotificationEmailAddresses; @@ -88,7 +91,9 @@ public class RegistryConfigSettings { public String reservedTermsExportDisclaimer; public String whoisDisclaimer; public String rdapTos; + public String rdapTosStaticUrl; public String spec11EmailBodyTemplate; + public boolean requireSslCertificates; } /** Configuration for Cloud Datastore. */ @@ -129,6 +134,7 @@ public class RegistryConfigSettings { /** Configuration for monthly invoices. */ public static class Billing { public List invoiceEmailRecipients; + public String invoiceFilePrefix; } /** Configuration for Registry Data Escrow (RDE). */ @@ -160,7 +166,6 @@ public class RegistryConfigSettings { public String sheetExportId; public String alertRecipientEmailAddress; public String spec11ReplyToEmailAddress; - public String alertEmailSenderDomain; public int asyncDeleteDelaySeconds; } @@ -178,6 +183,7 @@ public class RegistryConfigSettings { /** Configuration options for the registry tool. */ public static class RegistryTool { - public String clientSecretFilename; + public String clientId; + public String clientSecret; } } diff --git a/java/google/registry/config/RegistryEnvironment.java b/java/google/registry/config/RegistryEnvironment.java index 1c6c373ea..cc3b2a558 100644 --- a/java/google/registry/config/RegistryEnvironment.java +++ b/java/google/registry/config/RegistryEnvironment.java @@ -47,11 +47,25 @@ public enum RegistryEnvironment { */ UNITTEST; + /** Sets this enum as the name of the registry environment. */ + public RegistryEnvironment setup() { + return setup(SystemPropertySetter.PRODUCTION_IMPL); + } + + /** + * Sets this enum as the name of the registry environment using specified {@link + * SystemPropertySetter}. + */ + public RegistryEnvironment setup(SystemPropertySetter systemPropertySetter) { + systemPropertySetter.setProperty(PROPERTY, name()); + return this; + } + /** Returns environment configured by system property {@value #PROPERTY}. */ public static RegistryEnvironment get() { return valueOf(Ascii.toUpperCase(System.getProperty(PROPERTY, UNITTEST.name()))); } /** System property for configuring which environment we should use. */ - public static final String PROPERTY = "google.registry.environment"; + private static final String PROPERTY = "google.registry.environment"; } diff --git a/java/google/registry/config/SystemPropertySetter.java b/java/google/registry/config/SystemPropertySetter.java new file mode 100644 index 000000000..41cd864ea --- /dev/null +++ b/java/google/registry/config/SystemPropertySetter.java @@ -0,0 +1,47 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.config; + +import javax.annotation.Nullable; + +/** + * Wrapper interface around {@link System#setProperty(String, String)} and {@link + * System#clearProperty(String)}. Tests that modify system properties may provide custom + * implementations that keeps track of changes and restores original property values on test + * completion. + */ +public interface SystemPropertySetter { + + /** + * Updates the system property specified by {@code key}. If {@code value} is not null, {@link + * System#setProperty(String, String)} is invoked; otherwise {@link System#clearProperty(String)} + * is invoked. + */ + SystemPropertySetter setProperty(String key, @Nullable String value); + + /** Production implementation of {@link SystemPropertySetter}. */ + SystemPropertySetter PRODUCTION_IMPL = + new SystemPropertySetter() { + @Override + public SystemPropertySetter setProperty(String key, @Nullable String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + return this; + } + }; +} diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index 9c4a37628..17e2bbf19 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -22,6 +22,9 @@ gSuite: domainName: domain-registry.example # Display name and email address used on outgoing emails through G Suite. + # The email address must be valid and have permission in the GAE app to send + # emails. For more info see: + # https://cloud.google.com/appengine/docs/standard/java/mail/#who_can_send_mail outgoingEmailDisplayName: Example Registry outgoingEmailAddress: noreply@project-id.appspotmail.com @@ -29,6 +32,10 @@ gSuite: # logging in to perform administrative actions, not sending emails. adminAccountEmailAddress: admin@example.com + # Group containing the emails of the support accounts. These accounts will be + # given "ADMIN" role on the registrar console. + supportGroupEmailAddress: support@example.com + registryPolicy: # Repository identifier (ROID) suffix for contacts and hosts. contactAndHostRoidSuffix: ROID @@ -48,6 +55,10 @@ registryPolicy: # See flows/domain/token/AllocationTokenCustomLogic.java allocationTokenCustomLogicClass: google.registry.flows.domain.token.AllocationTokenCustomLogic + # Custom logic class for handling DNS query count reporting for ICANN. + # See reporting/icann/DnsCountQueryCoordinator.java + dnsCountQueryCoordinatorClass: google.registry.reporting.icann.BasicDnsCountQueryCoordinator + # Length of time after which contact transfers automatically conclude. contactAutomaticTransferDays: 5 @@ -145,6 +156,10 @@ registryPolicy: We reserve the right to modify this agreement at any time. + # Link to static Web page with RDAP terms of service. Displayed in RDAP + # responses. If null, no static Web page link is generated. + rdapTosStaticUrl: null + # Body of the spec 11 email sent to registrars. # Items in braces are to be replaced. spec11EmailBodyTemplate: | @@ -172,6 +187,11 @@ registryPolicy: If you have any questions regarding this notice, please contact {REPLY_TO_EMAIL}. + # Whether to require an SSL certificate hash in order to be able to log in + # via EPP and run commands. This can be false for testing environments but + # should generally be true for production environments, for added security. + requireSslCertificates: true + datastore: # Number of commit log buckets in Datastore. Lowering this after initial # install risks losing up to a days' worth of differential backups. @@ -186,13 +206,14 @@ datastore: baseOfyRetryMillis: 100 cloudDns: + # Set both properties to null in Production. # The root url for the Cloud DNS API. Set this to a non-null value to # override the default API server used by the googleapis library. - rootUrl: null + rootUrl: https://staging-www.sandbox.googleapis.com # The service endpoint path for the Cloud DNS API. Set this to a non-null # value to override the default API path used by the googleapis library. - servicePath: null + servicePath: dns/v2beta1_staging/projects/ caching: # Length of time that a singleton should be cached before expiring. @@ -245,8 +266,8 @@ oAuth: - https://www.googleapis.com/auth/userinfo.email # OAuth client IDs that are allowed to authenticate and communicate with - # backend services, e. g. nomulus tool, EPP proxy, etc. All client_id values - # used in client_secret.json files for associated tooling should be included + # backend services, e. g. nomulus tool, EPP proxy, etc. The client_id value + # used in registryTool.clientId field for associated tooling should be included # in this list. Client IDs are typically of the format # numbers-alphanumerics.apps.googleusercontent.com allowedOauthClientIds: [] @@ -271,7 +292,16 @@ credentialOAuth: - https://www.googleapis.com/auth/admin.directory.group # View and manage group settings in Group Settings API. - https://www.googleapis.com/auth/apps.groups.settings - + # OAuth scopes required to create a credential locally in for the nomulus tool. + localCredentialOauthScopes: + # View and manage data in all Google Cloud APIs. + - https://www.googleapis.com/auth/cloud-platform + # Call App Engine APIs locally. + - https://www.googleapis.com/auth/appengine.apis + # View your email address. + - https://www.googleapis.com/auth/userinfo.email + # View and manage your applications deployed on Google App Engine + - https://www.googleapis.com/auth/appengine.admin icannReporting: # URL we PUT monthly ICANN transactions reports to. @@ -282,6 +312,7 @@ icannReporting: billing: invoiceEmailRecipients: [] + invoiceFilePrefix: REG-INV rde: # URL prefix of ICANN's server to upload RDE reports to. Nomulus adds /TLD/ID @@ -339,19 +370,11 @@ misc: # to be a deliverable email address in case the registrars want to contact us. spec11ReplyToEmailAddress: reply-to@example.com - # Domain for the email address we send alert summary emails from. - alertEmailSenderDomain: appspotmail.com - # How long to delay processing of asynchronous deletions. This should always # be longer than eppResourceCachingSeconds, to prevent deleted contacts or # hosts from being used on domains. asyncDeleteDelaySeconds: 90 -cloudDns: - # CloudDns testing config. Set both properties to null in Production. - rootUrl: https://staging-www.sandbox.googleapis.com - servicePath: dns/v2beta1_staging/projects/ - beam: # The default zone to run Apache Beam (Cloud Dataflow) jobs in. defaultJobZone: us-east1-c @@ -372,5 +395,7 @@ keyring: # Configuration options relevant to the "nomulus" registry tool. registryTool: - # Name of the client secret file used for authenticating with App Engine. - clientSecretFilename: /google/registry/tools/resources/client_secret.json + # OAuth client Id used by the tool. + clientId: YOUR_CLIENT_ID + # OAuth client secret used by the tool. + clientSecret: YOUR_CLIENT_SECRET diff --git a/java/google/registry/config/files/nomulus-config-production-sample.yaml b/java/google/registry/config/files/nomulus-config-production-sample.yaml index 9d2e9ca44..341ccaeb2 100644 --- a/java/google/registry/config/files/nomulus-config-production-sample.yaml +++ b/java/google/registry/config/files/nomulus-config-production-sample.yaml @@ -18,6 +18,7 @@ gSuite: outgoingEmailDisplayName: placeholder outgoingEmailAddress: placeholder adminAccountEmailAddress: placeholder + supportGroupEmailAddress: placeholder registryPolicy: contactAndHostRoidSuffix: placeholder @@ -43,6 +44,7 @@ icannReporting: oAuth: allowedOauthClientIds: - placeholder.apps.googleusercontent.com + - placeholder-for-proxy rde: reportUrlPrefix: https://ry-api.icann.org/report/registry-escrow-report @@ -68,3 +70,7 @@ keyring: activeKeyring: KMS kms: projectId: placeholder + +registryTool: + clientId: placeholder.apps.googleusercontent.com + clientSecret: placeholder diff --git a/java/google/registry/config/files/nomulus-config-unittest.yaml b/java/google/registry/config/files/nomulus-config-unittest.yaml index a41fd9cf6..00501f26e 100644 --- a/java/google/registry/config/files/nomulus-config-unittest.yaml +++ b/java/google/registry/config/files/nomulus-config-unittest.yaml @@ -22,3 +22,8 @@ caching: staticPremiumListMaxCachedEntries: 50 eppResourceCachingEnabled: true eppResourceCachingSeconds: 0 + +# Remove the support G Suite group, because we don't want to try connecting to G Suite servers from +# tests +gSuite: + supportGroupEmailAddress: diff --git a/java/google/registry/dns/RefreshDnsAction.java b/java/google/registry/dns/RefreshDnsAction.java index 64920eb8b..f9c687a32 100644 --- a/java/google/registry/dns/RefreshDnsAction.java +++ b/java/google/registry/dns/RefreshDnsAction.java @@ -65,13 +65,14 @@ public final class RefreshDnsAction implements Runnable { private T loadAndVerifyExistence(Class clazz, String foreignKey) { - T resource = loadByForeignKey(clazz, foreignKey, clock.nowUtc()); - if (resource == null) { - String typeName = clazz.getAnnotation(ExternalMessagingName.class).value(); - throw new NotFoundException( - String.format("%s %s not found", typeName, domainOrHostName)); - } - return resource; + return loadByForeignKey(clazz, foreignKey, clock.nowUtc()) + .orElseThrow( + () -> + new NotFoundException( + String.format( + "%s %s not found", + clazz.getAnnotation(ExternalMessagingName.class).value(), + domainOrHostName))); } private static void verifyHostIsSubordinate(HostResource host) { diff --git a/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java b/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java index 5a9e61f95..152921277 100644 --- a/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java +++ b/java/google/registry/dns/writer/clouddns/CloudDnsWriter.java @@ -19,7 +19,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.util.DomainNameUtils.getSecondLevelDomain; -import com.google.api.client.googleapis.json.GoogleJsonError.ErrorInfo; +import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.services.dns.Dns; import com.google.api.services.dns.model.Change; @@ -120,9 +120,9 @@ public class CloudDnsWriter extends BaseDnsWriter { // Canonicalize name String absoluteDomainName = getAbsoluteHostName(domainName); - // Load the target domain. Note that it can be null if this domain was just deleted. + // Load the target domain. Note that it can be absent if this domain was just deleted. Optional domainResource = - Optional.ofNullable(loadByForeignKey(DomainResource.class, domainName, clock.nowUtc())); + loadByForeignKey(DomainResource.class, domainName, clock.nowUtc()); // Return early if no DNS records should be published. // desiredRecordsBuilder is populated with an empty set to indicate that all existing records @@ -188,11 +188,10 @@ public class CloudDnsWriter extends BaseDnsWriter { // Canonicalize name String absoluteHostName = getAbsoluteHostName(hostName); - // Load the target host. Note that it can be null if this host was just deleted. + // Load the target host. Note that it can be absent if this host was just deleted. // desiredRecords is populated with an empty set to indicate that all existing records // should be deleted. - Optional host = - Optional.ofNullable(loadByForeignKey(HostResource.class, hostName, clock.nowUtc())); + Optional host = loadByForeignKey(HostResource.class, hostName, clock.nowUtc()); // Return early if the host is deleted. if (!host.isPresent()) { @@ -390,12 +389,12 @@ public class CloudDnsWriter extends BaseDnsWriter { try { dnsConnection.changes().create(projectId, zoneName, change).execute(); } catch (GoogleJsonResponseException e) { - List errors = e.getDetails().getErrors(); + GoogleJsonError err = e.getDetails(); // We did something really wrong here, just give up and re-throw - if (errors.size() > 1) { + if (err == null || err.getErrors().size() > 1) { throw new RuntimeException(e); } - String errorReason = errors.get(0).getReason(); + String errorReason = err.getErrors().get(0).getReason(); if (RETRYABLE_EXCEPTION_REASONS.contains(errorReason)) { throw new ZoneStateException(errorReason); diff --git a/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriter.java b/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriter.java index c20813dd6..4baecb065 100644 --- a/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriter.java +++ b/java/google/registry/dns/writer/dnsupdate/DnsUpdateWriter.java @@ -14,6 +14,7 @@ package google.registry.dns.writer.dnsupdate; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static com.google.common.collect.Sets.intersection; import static com.google.common.collect.Sets.union; @@ -126,9 +127,12 @@ public class DnsUpdateWriter extends BaseDnsWriter { * this domain refresh request */ private void publishDomain(String domainName, String requestingHostName) { - DomainResource domain = loadByForeignKey(DomainResource.class, domainName, clock.nowUtc()); + Optional domainOptional = + loadByForeignKey(DomainResource.class, domainName, clock.nowUtc()); update.delete(toAbsoluteName(domainName), Type.ANY); - if (domain != null) { + // If the domain is now deleted, then don't update DNS for it. + if (domainOptional.isPresent()) { + DomainResource domain = domainOptional.get(); // As long as the domain exists, orphan glues should be cleaned. deleteSubordinateHostAddressSet(domain, requestingHostName, update); if (domain.shouldPublishToDns()) { @@ -213,9 +217,10 @@ public class DnsUpdateWriter extends BaseDnsWriter { for (String hostName : intersection( domain.loadNameserverFullyQualifiedHostNames(), domain.getSubordinateHosts())) { - HostResource host = loadByForeignKey(HostResource.class, hostName, clock.nowUtc()); - update.add(makeAddressSet(host)); - update.add(makeV6AddressSet(host)); + Optional host = loadByForeignKey(HostResource.class, hostName, clock.nowUtc()); + checkState(host.isPresent(), "Host %s cannot be loaded", hostName); + update.add(makeAddressSet(host.get())); + update.add(makeV6AddressSet(host.get())); } } diff --git a/java/google/registry/env/alpha/backend/WEB-INF/appengine-web.xml b/java/google/registry/env/alpha/backend/WEB-INF/appengine-web.xml index 9d026e1ce..82d84abad 100644 --- a/java/google/registry/env/alpha/backend/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/alpha/backend/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - backend + backend true true B4 diff --git a/java/google/registry/env/alpha/default/WEB-INF/appengine-web.xml b/java/google/registry/env/alpha/default/WEB-INF/appengine-web.xml index f93787fb5..dfe7ac4d5 100644 --- a/java/google/registry/env/alpha/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/alpha/default/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - default + default true true B4 diff --git a/java/google/registry/env/alpha/default/WEB-INF/cron.xml b/java/google/registry/env/alpha/default/WEB-INF/cron.xml index 086cba362..845822129 100644 --- a/java/google/registry/env/alpha/default/WEB-INF/cron.xml +++ b/java/google/registry/env/alpha/default/WEB-INF/cron.xml @@ -142,6 +142,24 @@ backend + + + + This job fires off a Datastore managed-export job that generates snapshot files in GCS. + It also enqueues a new task to wait on the completion of that job and then load the resulting + snapshot into bigquery. + + + every day 09:00 + backend + + diff --git a/java/google/registry/env/alpha/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/alpha/pubapi/WEB-INF/appengine-web.xml index ca8587338..914df6906 100644 --- a/java/google/registry/env/alpha/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/alpha/pubapi/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - pubapi + pubapi true true B4 diff --git a/java/google/registry/env/alpha/tools/WEB-INF/appengine-web.xml b/java/google/registry/env/alpha/tools/WEB-INF/appengine-web.xml index bef98ba66..eb0e5a938 100644 --- a/java/google/registry/env/alpha/tools/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/alpha/tools/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - tools + tools true true B4 diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index 2092c4782..b80efaa01 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -174,6 +174,24 @@ /_dr/dnsRefresh + + + backend-servlet + /_dr/task/backupDatastore + + + + + backend-servlet + /_dr/task/checkDatastoreBackup + + + + + backend-servlet + /_dr/task/uploadDatastoreBackup + + backend-servlet diff --git a/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml b/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml index 26f3416dc..481129a1a 100644 --- a/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml +++ b/java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml @@ -69,10 +69,14 @@ - + + + + + diff --git a/java/google/registry/env/common/default/WEB-INF/web.xml b/java/google/registry/env/common/default/WEB-INF/web.xml index ad42c0f06..bcbc77703 100644 --- a/java/google/registry/env/common/default/WEB-INF/web.xml +++ b/java/google/registry/env/common/default/WEB-INF/web.xml @@ -37,6 +37,12 @@ /registrar-settings + + + frontend-servlet + /registrar-ote-setup + + diff --git a/java/google/registry/env/common/tools/WEB-INF/web.xml b/java/google/registry/env/common/tools/WEB-INF/web.xml index eb7dde515..0bd9341d1 100644 --- a/java/google/registry/env/common/tools/WEB-INF/web.xml +++ b/java/google/registry/env/common/tools/WEB-INF/web.xml @@ -72,6 +72,12 @@ /_dr/task/resaveAllHistoryEntries + + + tools-servlet + /_dr/task/killAllDomainApplications + + tools-servlet diff --git a/java/google/registry/env/crash/backend/WEB-INF/appengine-web.xml b/java/google/registry/env/crash/backend/WEB-INF/appengine-web.xml index 47aa68a9a..e6070b5f0 100644 --- a/java/google/registry/env/crash/backend/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/crash/backend/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - backend + backend true true B4 diff --git a/java/google/registry/env/crash/default/WEB-INF/appengine-web.xml b/java/google/registry/env/crash/default/WEB-INF/appengine-web.xml index 7ddffbd5d..fe8cff576 100644 --- a/java/google/registry/env/crash/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/crash/default/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - default + default true true B4_1G diff --git a/java/google/registry/env/crash/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/crash/pubapi/WEB-INF/appengine-web.xml index 47a7f6edf..c6c738bae 100644 --- a/java/google/registry/env/crash/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/crash/pubapi/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - pubapi + pubapi true true B4 diff --git a/java/google/registry/env/crash/tools/WEB-INF/appengine-web.xml b/java/google/registry/env/crash/tools/WEB-INF/appengine-web.xml index de738b6cd..065d3624d 100644 --- a/java/google/registry/env/crash/tools/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/crash/tools/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - tools + tools true true B4 diff --git a/java/google/registry/env/local/backend/WEB-INF/appengine-web.xml b/java/google/registry/env/local/backend/WEB-INF/appengine-web.xml index dc2f284d1..2cad41736 100644 --- a/java/google/registry/env/local/backend/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/local/backend/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - backend + backend true true B4 diff --git a/java/google/registry/env/local/default/WEB-INF/appengine-web.xml b/java/google/registry/env/local/default/WEB-INF/appengine-web.xml index ab0a2ccac..361ba13e9 100644 --- a/java/google/registry/env/local/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/local/default/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - default + default true true B4_1G diff --git a/java/google/registry/env/local/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/local/pubapi/WEB-INF/appengine-web.xml index a26210a8a..39bae1a9c 100644 --- a/java/google/registry/env/local/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/local/pubapi/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - pubapi + pubapi true true B4 diff --git a/java/google/registry/env/local/tools/WEB-INF/appengine-web.xml b/java/google/registry/env/local/tools/WEB-INF/appengine-web.xml index 620019593..037d5769c 100644 --- a/java/google/registry/env/local/tools/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/local/tools/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - tools + tools true true B4 diff --git a/java/google/registry/env/production/backend/WEB-INF/appengine-web.xml b/java/google/registry/env/production/backend/WEB-INF/appengine-web.xml index 244c1f043..d19733cd6 100644 --- a/java/google/registry/env/production/backend/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/production/backend/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - backend + backend true true B4_1G diff --git a/java/google/registry/env/production/default/WEB-INF/appengine-web.xml b/java/google/registry/env/production/default/WEB-INF/appengine-web.xml index ef28324b7..7449a161c 100644 --- a/java/google/registry/env/production/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/production/default/WEB-INF/appengine-web.xml @@ -1,15 +1,13 @@ - domain-registry - 1 java8 - default + default true true B4_1G - 30 + 15 diff --git a/java/google/registry/env/production/default/WEB-INF/cron.xml b/java/google/registry/env/production/default/WEB-INF/cron.xml index d4f27477f..cf6090023 100644 --- a/java/google/registry/env/production/default/WEB-INF/cron.xml +++ b/java/google/registry/env/production/default/WEB-INF/cron.xml @@ -174,6 +174,24 @@ backend + + + + This job fires off a Datastore managed-export job that generates snapshot files in GCS. + It also enqueues a new task to wait on the completion of that job and then load the resulting + snapshot into bigquery. + + + every day 09:00 + backend + + @@ -289,13 +307,12 @@ - Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates last month's Spec11 - report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM. - Upon Dataflow job completion, sends an e-mail to all registrars with domain registrations - flagged by the SafeBrowsing API. + Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates today's Spec11 + report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM/. + This job will only send email notifications on the second of every month. See GenerateSpec11ReportAction for more details. - 2 of month 15:00 + every day 15:00 backend diff --git a/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml index 5a6ea5c8e..de2869151 100644 --- a/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/production/pubapi/WEB-INF/appengine-web.xml @@ -1,15 +1,13 @@ - domain-registry - 1 java8 - pubapi + pubapi true true B4_1G - 20 + 10 diff --git a/java/google/registry/env/production/tools/WEB-INF/appengine-web.xml b/java/google/registry/env/production/tools/WEB-INF/appengine-web.xml index 1d02a70a7..8ce6932fa 100644 --- a/java/google/registry/env/production/tools/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/production/tools/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - tools + tools true true B4_1G diff --git a/java/google/registry/env/qa/backend/WEB-INF/appengine-web.xml b/java/google/registry/env/qa/backend/WEB-INF/appengine-web.xml index e75dfe585..f83f825b3 100644 --- a/java/google/registry/env/qa/backend/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/qa/backend/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - backend + backend true true B4 diff --git a/java/google/registry/env/qa/default/WEB-INF/appengine-web.xml b/java/google/registry/env/qa/default/WEB-INF/appengine-web.xml index c79f718ab..0eeb4341c 100644 --- a/java/google/registry/env/qa/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/qa/default/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - default + default true true F4_1G diff --git a/java/google/registry/env/qa/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/qa/pubapi/WEB-INF/appengine-web.xml index 136afec07..10fb323de 100644 --- a/java/google/registry/env/qa/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/qa/pubapi/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - pubapi + pubapi true true B4 diff --git a/java/google/registry/env/qa/tools/WEB-INF/appengine-web.xml b/java/google/registry/env/qa/tools/WEB-INF/appengine-web.xml index 1e48c3e22..778d623e9 100644 --- a/java/google/registry/env/qa/tools/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/qa/tools/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - tools + tools true true B4 diff --git a/java/google/registry/env/sandbox/backend/WEB-INF/appengine-web.xml b/java/google/registry/env/sandbox/backend/WEB-INF/appengine-web.xml index 16cb0dd71..468720dbf 100644 --- a/java/google/registry/env/sandbox/backend/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/sandbox/backend/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - backend + backend true true B4 diff --git a/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml b/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml index c5ed1090a..2cd7af386 100644 --- a/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/sandbox/default/WEB-INF/appengine-web.xml @@ -1,15 +1,13 @@ - domain-registry - 1 java8 - default + default true true B4_1G - 30 + 10 diff --git a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml index 9ed16a801..a02d57f7c 100644 --- a/java/google/registry/env/sandbox/default/WEB-INF/cron.xml +++ b/java/google/registry/env/sandbox/default/WEB-INF/cron.xml @@ -151,6 +151,24 @@ backend + + + + This job fires off a Datastore managed-export job that generates snapshot files in GCS. + It also enqueues a new task to wait on the completion of that job and then load the resulting + snapshot into bigquery. + + + every day 09:00 + backend + + diff --git a/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml b/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml index cde9e57a8..e82dd364c 100644 --- a/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/sandbox/pubapi/WEB-INF/appengine-web.xml @@ -1,15 +1,13 @@ - domain-registry - 1 java8 - pubapi + pubapi true true B4_1G - 20 + 5 diff --git a/java/google/registry/env/sandbox/tools/WEB-INF/appengine-web.xml b/java/google/registry/env/sandbox/tools/WEB-INF/appengine-web.xml index eaa2469fd..55a950088 100644 --- a/java/google/registry/env/sandbox/tools/WEB-INF/appengine-web.xml +++ b/java/google/registry/env/sandbox/tools/WEB-INF/appengine-web.xml @@ -1,10 +1,8 @@ - domain-registry - 1 java8 - tools + tools true true B4 diff --git a/java/google/registry/export/BUILD b/java/google/registry/export/BUILD index 5ad934b5d..91237a800 100644 --- a/java/google/registry/export/BUILD +++ b/java/google/registry/export/BUILD @@ -10,6 +10,7 @@ java_library( deps = [ "//java/google/registry/bigquery", "//java/google/registry/config", + "//java/google/registry/export/datastore", "//java/google/registry/gcs", "//java/google/registry/groups", "//java/google/registry/mapreduce", diff --git a/java/google/registry/export/BackupDatastoreAction.java b/java/google/registry/export/BackupDatastoreAction.java new file mode 100644 index 000000000..bd1eb1146 --- /dev/null +++ b/java/google/registry/export/BackupDatastoreAction.java @@ -0,0 +1,86 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static google.registry.export.CheckBackupAction.enqueuePollTask; +import static google.registry.request.Action.Method.POST; + +import com.google.common.flogger.FluentLogger; +import google.registry.config.RegistryConfig; +import google.registry.export.datastore.DatastoreAdmin; +import google.registry.export.datastore.Operation; +import google.registry.request.Action; +import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import javax.inject.Inject; + +/** + * Action to trigger a Datastore backup job that writes a snapshot to Google Cloud Storage. This + * class is introduced as an experimental feature, and will eventually replace {@link + * ExportSnapshotAction}. + * + *

This is the first step of a four step workflow for exporting snapshots, with each step calling + * the next upon successful completion: + * + *

    + *
  1. The snapshot is exported to Google Cloud Storage (this action). + *
  2. The {@link CheckBackupAction} polls until the export is completed. + *
  3. The {@link UploadDatastoreBackupAction} uploads the data from GCS to BigQuery. + *
  4. The {@link UpdateSnapshotViewAction} updates the view in latest_datastore_export. + *
+ */ +@Action( + path = BackupDatastoreAction.PATH, + method = POST, + automaticallyPrintOk = true, + auth = Auth.AUTH_INTERNAL_ONLY) +public class BackupDatastoreAction implements Runnable { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + /** Queue to use for enqueuing the task that will actually launch the backup. */ + static final String QUEUE = "export-snapshot"; // See queue.xml. + + static final String PATH = "/_dr/task/backupDatastore"; // See web.xml. + + @Inject DatastoreAdmin datastoreAdmin; + @Inject Response response; + + @Inject + BackupDatastoreAction() {} + + @Override + public void run() { + try { + Operation backup = + datastoreAdmin + .export(RegistryConfig.getDatastoreBackupsBucket(), ExportConstants.getBackupKinds()) + .execute(); + + String backupName = backup.getName(); + // Enqueue a poll task to monitor the backup and load REPORTING-related kinds into bigquery. + enqueuePollTask(backupName, ExportConstants.getReportingKinds()); + String message = + String.format( + "Datastore backup started with name: %s\nSaving to %s", + backupName, backup.getExportFolderUrl()); + logger.atInfo().log(message); + response.setPayload(message); + } catch (Throwable e) { + throw new InternalServerErrorException("Exception occurred while backing up datastore.", e); + } + } +} diff --git a/java/google/registry/export/CheckBackupAction.java b/java/google/registry/export/CheckBackupAction.java new file mode 100644 index 000000000..3ef56da79 --- /dev/null +++ b/java/google/registry/export/CheckBackupAction.java @@ -0,0 +1,189 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.collect.Sets.intersection; +import static google.registry.export.UploadDatastoreBackupAction.enqueueUploadBackupTask; +import static google.registry.request.Action.Method.GET; +import static google.registry.request.Action.Method.POST; +import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; + +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskHandle; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.appengine.api.taskqueue.TaskOptions.Method; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.flogger.FluentLogger; +import google.registry.export.datastore.DatastoreAdmin; +import google.registry.export.datastore.Operation; +import google.registry.request.Action; +import google.registry.request.HttpException; +import google.registry.request.HttpException.BadRequestException; +import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.request.HttpException.NoContentException; +import google.registry.request.HttpException.NotModifiedException; +import google.registry.request.Parameter; +import google.registry.request.RequestMethod; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.util.Clock; +import java.io.IOException; +import java.util.Set; +import javax.inject.Inject; +import org.joda.time.Duration; +import org.joda.time.PeriodType; +import org.joda.time.format.PeriodFormat; + +/** + * Action that checks the status of a snapshot, and if complete, trigger loading it into BigQuery. + */ +@Action( + path = CheckBackupAction.PATH, + method = {POST, GET}, + automaticallyPrintOk = true, + auth = Auth.AUTH_INTERNAL_ONLY) +public class CheckBackupAction implements Runnable { + + /** Parameter names for passing parameters into this action. */ + static final String CHECK_BACKUP_NAME_PARAM = "name"; + + static final String CHECK_BACKUP_KINDS_TO_LOAD_PARAM = "kindsToLoad"; + + /** Action-specific details needed for enqueuing tasks against itself. */ + static final String QUEUE = "export-snapshot-poll"; // See queue.xml. + + static final String PATH = "/_dr/task/checkDatastoreBackup"; // See web.xml. + static final Duration POLL_COUNTDOWN = Duration.standardMinutes(2); + + /** The maximum amount of time we allow a backup to run before abandoning it. */ + static final Duration MAXIMUM_BACKUP_RUNNING_TIME = Duration.standardHours(20); + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Inject DatastoreAdmin datastoreAdmin; + @Inject Clock clock; + @Inject Response response; + @Inject @RequestMethod Action.Method requestMethod; + + @Inject + @Parameter(CHECK_BACKUP_NAME_PARAM) + String backupName; + + @Inject + @Parameter(CHECK_BACKUP_KINDS_TO_LOAD_PARAM) + String kindsToLoadParam; + + @Inject + CheckBackupAction() {} + + @Override + public void run() { + try { + if (requestMethod == POST) { + checkAndLoadBackupIfComplete(); + } else { + // This is a GET request. + // TODO(weiminyu): consider moving this functionality to Registry tool. + response.setPayload(getExportStatus().toPrettyString()); + } + } catch (HttpException e) { + // Rethrow and let caller propagate status code and error message to the response. + // See google.registry.request.RequestHandler#handleRequest. + throw e; + } catch (Throwable e) { + throw new InternalServerErrorException( + String.format("Exception occurred while checking datastore exports."), e); + } + } + + private Operation getExportStatus() throws IOException { + try { + return datastoreAdmin.get(backupName).execute(); + } catch (GoogleJsonResponseException e) { + if (e.getStatusCode() == SC_NOT_FOUND) { + String message = String.format("Bad backup name %s: %s", backupName, e.getMessage()); + // TODO(b/19081569): Ideally this would return a 2XX error so the task would not be + // retried but we might abandon backups that start late and haven't yet written to + // Datastore. We could fix that by replacing this with a two-phase polling strategy. + throw new BadRequestException(message, e); + } + throw e; + } + } + + private void checkAndLoadBackupIfComplete() throws IOException { + Set kindsToLoad = ImmutableSet.copyOf(Splitter.on(',').split(kindsToLoadParam)); + Operation backup = getExportStatus(); + + if (backup.isProcessing() + && backup.getRunningTime(clock).isShorterThan(MAXIMUM_BACKUP_RUNNING_TIME)) { + // Backup might still be running, so send a 304 to have the task retry. + throw new NotModifiedException( + String.format( + "Datastore backup %s still in progress: %s", backupName, backup.getProgress())); + } + if (!backup.isSuccessful()) { + // Declare the backup a lost cause, and send 204 No Content so the task will + // not be retried. + String message = + String.format( + "Datastore backup %s abandoned - not complete after %s. Progress: %s", + backupName, + PeriodFormat.getDefault() + .print( + backup + .getRunningTime(clock) + .toPeriod() + .normalizedStandard(PeriodType.dayTime().withMillisRemoved())), + backup.getProgress()); + throw new NoContentException(message); + } + + String backupId = backup.getExportId(); + // Log a warning if kindsToLoad is not a subset of the exported kinds. + if (!backup.getKinds().containsAll(kindsToLoad)) { + logger.atWarning().log( + "Kinds to load included non-exported kinds: %s", + Sets.difference(kindsToLoad, backup.getKinds())); + } + // Load kinds from the backup, limited to those also in kindsToLoad (if it's present). + ImmutableSet exportedKindsToLoad = + ImmutableSet.copyOf(intersection(backup.getKinds(), kindsToLoad)); + String message = String.format("Datastore backup %s complete - ", backupName); + if (exportedKindsToLoad.isEmpty()) { + message += "no kinds to load into BigQuery"; + } else { + enqueueUploadBackupTask(backupId, backup.getExportFolderUrl(), exportedKindsToLoad); + message += "BigQuery load task enqueued"; + } + logger.atInfo().log(message); + response.setPayload(message); + } + + /** Enqueue a poll task to monitor the named backup for completion. */ + static TaskHandle enqueuePollTask(String backupId, ImmutableSet kindsToLoad) { + return QueueFactory.getQueue(QUEUE) + .add( + TaskOptions.Builder.withUrl(PATH) + .method(Method.POST) + .countdownMillis(POLL_COUNTDOWN.getMillis()) + .param(CHECK_BACKUP_NAME_PARAM, backupId) + .param(CHECK_BACKUP_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad))); + } +} diff --git a/java/google/registry/export/ExportRequestModule.java b/java/google/registry/export/ExportRequestModule.java index abb14efd6..e1e7148cf 100644 --- a/java/google/registry/export/ExportRequestModule.java +++ b/java/google/registry/export/ExportRequestModule.java @@ -25,6 +25,8 @@ import static google.registry.export.LoadSnapshotAction.LOAD_SNAPSHOT_KINDS_PARA import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM; import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM; +import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM; +import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM; import static google.registry.request.RequestParameters.extractRequiredHeader; import static google.registry.request.RequestParameters.extractRequiredParameter; @@ -56,12 +58,24 @@ public final class ExportRequestModule { return extractRequiredParameter(req, UPDATE_SNAPSHOT_KIND_PARAM); } + @Provides + @Parameter(UPDATE_SNAPSHOT_VIEWNAME_PARAM) + static String provideUpdateSnapshotViewName(HttpServletRequest req) { + return extractRequiredParameter(req, UPDATE_SNAPSHOT_VIEWNAME_PARAM); + } + @Provides @Parameter(LOAD_SNAPSHOT_FILE_PARAM) static String provideLoadSnapshotFile(HttpServletRequest req) { return extractRequiredParameter(req, LOAD_SNAPSHOT_FILE_PARAM); } + @Provides + @Parameter(UPLOAD_BACKUP_FOLDER_PARAM) + static String provideSnapshotUrlPrefix(HttpServletRequest req) { + return extractRequiredParameter(req, UPLOAD_BACKUP_FOLDER_PARAM); + } + @Provides @Parameter(LOAD_SNAPSHOT_ID_PARAM) static String provideLoadSnapshotId(HttpServletRequest req) { diff --git a/java/google/registry/export/LoadSnapshotAction.java b/java/google/registry/export/LoadSnapshotAction.java index 5a4c2835e..91834b7d2 100644 --- a/java/google/registry/export/LoadSnapshotAction.java +++ b/java/google/registry/export/LoadSnapshotAction.java @@ -64,6 +64,8 @@ public class LoadSnapshotAction implements Runnable { static final String SNAPSHOTS_DATASET = "snapshots"; + static final String LATEST_SNAPSHOT_VIEW_NAME = "latest_datastore_export"; + /** Servlet-specific details needed for enqueuing tasks against itself. */ static final String QUEUE = "export-snapshot"; // See queue.xml. static final String PATH = "/_dr/task/loadSnapshot"; // See web.xml. @@ -131,7 +133,7 @@ public class LoadSnapshotAction implements Runnable { // well-known view in BigQuery to point at the newly loaded snapshot table for this kind. bigqueryPollEnqueuer.enqueuePollTask( jobRef, - createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName), + createViewUpdateTask(SNAPSHOTS_DATASET, tableId, kindName, LATEST_SNAPSHOT_VIEW_NAME), getQueue(UpdateSnapshotViewAction.QUEUE)); builder.append(String.format(" - %s:%s\n", projectId, jobId)); diff --git a/java/google/registry/export/UpdateSnapshotViewAction.java b/java/google/registry/export/UpdateSnapshotViewAction.java index cd2622b19..e792882f6 100644 --- a/java/google/registry/export/UpdateSnapshotViewAction.java +++ b/java/google/registry/export/UpdateSnapshotViewAction.java @@ -43,10 +43,12 @@ public class UpdateSnapshotViewAction implements Runnable { static final String UPDATE_SNAPSHOT_TABLE_ID_PARAM = "table"; static final String UPDATE_SNAPSHOT_KIND_PARAM = "kind"; - - private static final String TARGET_DATASET_NAME = "latest_datastore_export"; + static final String UPDATE_SNAPSHOT_VIEWNAME_PARAM = "viewname"; /** Servlet-specific details needed for enqueuing tasks against itself. */ + // For now this queue is shared by the backup workflows started by both ExportSnapshotAction + // and BackupDatastoreAction. + // TODO(weiminyu): update queue name (snapshot->backup) after ExportSnapshot flow is removed. static final String QUEUE = "export-snapshot-update-view"; // See queue.xml. static final String PATH = "/_dr/task/updateSnapshotView"; // See web.xml. @@ -65,6 +67,10 @@ public class UpdateSnapshotViewAction implements Runnable { @Parameter(UPDATE_SNAPSHOT_KIND_PARAM) String kindName; + @Inject + @Parameter(UPDATE_SNAPSHOT_VIEWNAME_PARAM) + String viewName; + @Inject @Config("projectId") String projectId; @@ -75,12 +81,14 @@ public class UpdateSnapshotViewAction implements Runnable { UpdateSnapshotViewAction() {} /** Create a task for updating a snapshot view. */ - static TaskOptions createViewUpdateTask(String datasetId, String tableId, String kindName) { + static TaskOptions createViewUpdateTask( + String datasetId, String tableId, String kindName, String viewName) { return TaskOptions.Builder.withUrl(PATH) .method(Method.POST) .param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, datasetId) .param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, tableId) - .param(UPDATE_SNAPSHOT_KIND_PARAM, kindName); + .param(UPDATE_SNAPSHOT_KIND_PARAM, kindName) + .param(UPDATE_SNAPSHOT_VIEWNAME_PARAM, viewName); } @Override @@ -89,12 +97,10 @@ public class UpdateSnapshotViewAction implements Runnable { SqlTemplate sqlTemplate = SqlTemplate.create( "#standardSQL\nSELECT * FROM `%PROJECT%.%SOURCE_DATASET%.%SOURCE_TABLE%`"); - updateSnapshotView(datasetId, tableId, kindName, TARGET_DATASET_NAME, sqlTemplate); + updateSnapshotView(datasetId, tableId, kindName, viewName, sqlTemplate); } catch (Throwable e) { throw new InternalServerErrorException( - String.format( - "Could not update snapshot view %s for table %s", TARGET_DATASET_NAME, tableId), - e); + String.format("Could not update snapshot view %s for table %s", viewName, tableId), e); } } @@ -138,7 +144,7 @@ public class UpdateSnapshotViewAction implements Runnable { .update(ref.getProjectId(), ref.getDatasetId(), ref.getTableId(), table) .execute(); } catch (GoogleJsonResponseException e) { - if (e.getDetails().getCode() == 404) { + if (e.getDetails() != null && e.getDetails().getCode() == 404) { bigquery.tables().insert(ref.getProjectId(), ref.getDatasetId(), table).execute(); } else { logger.atWarning().withCause(e).log( diff --git a/java/google/registry/export/UploadDatastoreBackupAction.java b/java/google/registry/export/UploadDatastoreBackupAction.java new file mode 100644 index 000000000..ce2504ba8 --- /dev/null +++ b/java/google/registry/export/UploadDatastoreBackupAction.java @@ -0,0 +1,181 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; +import static com.google.common.base.MoreObjects.firstNonNull; +import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask; +import static google.registry.request.Action.Method.POST; + +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.model.Job; +import com.google.api.services.bigquery.model.JobConfiguration; +import com.google.api.services.bigquery.model.JobConfigurationLoad; +import com.google.api.services.bigquery.model.JobReference; +import com.google.api.services.bigquery.model.TableReference; +import com.google.appengine.api.taskqueue.TaskHandle; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.appengine.api.taskqueue.TaskOptions.Method; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import google.registry.bigquery.BigqueryUtils.SourceFormat; +import google.registry.bigquery.BigqueryUtils.WriteDisposition; +import google.registry.bigquery.CheckedBigquery; +import google.registry.config.RegistryConfig.Config; +import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer; +import google.registry.request.Action; +import google.registry.request.HttpException.BadRequestException; +import google.registry.request.HttpException.InternalServerErrorException; +import google.registry.request.Parameter; +import google.registry.request.auth.Auth; +import java.io.IOException; +import javax.inject.Inject; + +/** Action to load a Datastore backup from Google Cloud Storage into BigQuery. */ +@Action(path = UploadDatastoreBackupAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_ONLY) +public class UploadDatastoreBackupAction implements Runnable { + + /** Parameter names for passing parameters into the servlet. */ + static final String UPLOAD_BACKUP_ID_PARAM = "id"; + + static final String UPLOAD_BACKUP_FOLDER_PARAM = "folder"; + static final String UPLOAD_BACKUP_KINDS_PARAM = "kinds"; + + static final String BACKUP_DATASET = "datastore_backups"; + + /** Servlet-specific details needed for enqueuing tasks against itself. */ + static final String QUEUE = "export-snapshot"; // See queue.xml. + + static final String LATEST_BACKUP_VIEW_NAME = "latest_datastore_backup"; + + static final String PATH = "/_dr/task/uploadDatastoreBackup"; // See web.xml. + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Inject CheckedBigquery checkedBigquery; + @Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer; + @Inject @Config("projectId") String projectId; + + @Inject + @Parameter(UPLOAD_BACKUP_FOLDER_PARAM) + String backupFolderUrl; + + @Inject + @Parameter(UPLOAD_BACKUP_ID_PARAM) + String backupId; + + @Inject + @Parameter(UPLOAD_BACKUP_KINDS_PARAM) + String backupKinds; + + @Inject + UploadDatastoreBackupAction() {} + + /** Enqueue a task for starting a backup load. */ + public static TaskHandle enqueueUploadBackupTask( + String backupId, String gcsFile, ImmutableSet kinds) { + return getQueue(QUEUE) + .add( + TaskOptions.Builder.withUrl(PATH) + .method(Method.POST) + .param(UPLOAD_BACKUP_ID_PARAM, backupId) + .param(UPLOAD_BACKUP_FOLDER_PARAM, gcsFile) + .param(UPLOAD_BACKUP_KINDS_PARAM, Joiner.on(',').join(kinds))); + } + + @Override + public void run() { + try { + String message = uploadBackup(backupId, backupFolderUrl, Splitter.on(',').split(backupKinds)); + logger.atInfo().log("Loaded backup successfully: %s", message); + } catch (Throwable e) { + logger.atSevere().withCause(e).log("Error loading backup"); + if (e instanceof IllegalArgumentException) { + throw new BadRequestException("Error calling load backup: " + e.getMessage(), e); + } else { + throw new InternalServerErrorException( + "Error loading backup: " + firstNonNull(e.getMessage(), e.toString())); + } + } + } + + private String uploadBackup(String backupId, String backupFolderUrl, Iterable kinds) + throws IOException { + Bigquery bigquery = checkedBigquery.ensureDataSetExists(projectId, BACKUP_DATASET); + String loadMessage = + String.format("Loading Datastore backup %s from %s...", backupId, backupFolderUrl); + logger.atInfo().log(loadMessage); + + String sanitizedBackupId = sanitizeForBigquery(backupId); + StringBuilder builder = new StringBuilder(loadMessage + "\n"); + builder.append("Load jobs:\n"); + + for (String kindName : kinds) { + String jobId = String.format("load-backup-%s-%s", sanitizedBackupId, kindName); + JobReference jobRef = new JobReference().setProjectId(projectId).setJobId(jobId); + String sourceUri = getBackupInfoFileForKind(backupFolderUrl, kindName); + String tableId = String.format("%s_%s", sanitizedBackupId, kindName); + + // Launch the load job. + Job job = makeLoadJob(jobRef, sourceUri, tableId); + bigquery.jobs().insert(projectId, job).execute(); + + // Enqueue a task to check on the load job's completion, and if it succeeds, to update a + // well-known view in BigQuery to point at the newly loaded backup table for this kind. + bigqueryPollEnqueuer.enqueuePollTask( + jobRef, + createViewUpdateTask(BACKUP_DATASET, tableId, kindName, LATEST_BACKUP_VIEW_NAME), + getQueue(UpdateSnapshotViewAction.QUEUE)); + + builder.append(String.format(" - %s:%s\n", projectId, jobId)); + logger.atInfo().log("Submitted load job %s:%s", projectId, jobId); + } + return builder.toString(); + } + + static String sanitizeForBigquery(String backupId) { + return backupId.replaceAll("[^a-zA-Z0-9_]", "_"); + } + + @VisibleForTesting + static String getBackupInfoFileForKind(String backupFolderUrl, String kindName) { + return Joiner.on('/') + .join( + backupFolderUrl, + "all_namespaces", + String.format("kind_%s", kindName), + String.format("all_namespaces_kind_%s.%s", kindName, "export_metadata")); + } + + private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) { + TableReference tableReference = + new TableReference() + .setProjectId(jobRef.getProjectId()) + .setDatasetId(BACKUP_DATASET) + .setTableId(tableId); + return new Job() + .setJobReference(jobRef) + .setConfiguration(new JobConfiguration() + .setLoad(new JobConfigurationLoad() + .setWriteDisposition(WriteDisposition.WRITE_EMPTY.toString()) + .setSourceFormat(SourceFormat.DATASTORE_BACKUP.toString()) + .setSourceUris(ImmutableList.of(sourceUri)) + .setDestinationTable(tableReference))); + } +} diff --git a/java/google/registry/export/datastore/BUILD b/java/google/registry/export/datastore/BUILD new file mode 100644 index 000000000..adc95e5a9 --- /dev/null +++ b/java/google/registry/export/datastore/BUILD @@ -0,0 +1,21 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +java_library( + name = "datastore", + srcs = glob(["*.java"]), + deps = [ + "//java/google/registry/config", + "//java/google/registry/util", + "@com_google_api_client", + "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_guava", + "@com_google_http_client", + "@com_google_http_client_jackson2", + "@joda_time", + ], +) diff --git a/java/google/registry/export/datastore/DatastoreAdmin.java b/java/google/registry/export/datastore/DatastoreAdmin.java new file mode 100644 index 000000000..4e21495f6 --- /dev/null +++ b/java/google/registry/export/datastore/DatastoreAdmin.java @@ -0,0 +1,223 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export.datastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.client.googleapis.services.json.AbstractGoogleJsonClient; +import com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.GenericJson; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.util.Key; +import com.google.common.base.Strings; +import java.util.Collection; +import java.util.Optional; + +/** + * Java client to Cloud + * Datastore Admin REST API. + */ +public class DatastoreAdmin extends AbstractGoogleJsonClient { + + private static final String ROOT_URL = "https://datastore.googleapis.com/v1/"; + private static final String SERVICE_PATH = ""; + + // GCP project that this instance is associated with. + private final String projectId; + + protected DatastoreAdmin(Builder builder) { + super(builder); + this.projectId = checkNotNull(builder.projectId, "GCP projectId missing."); + } + + /** + * Returns an {@link Export} request that starts exporting all Cloud Datastore databases owned by + * the GCP project identified by {@link #projectId}. + * + *

Typical usage is: + * + * 

+   *     {@code Export export = datastoreAdmin.export(parameters ...);}
+   *     {@code Operation operation = export.execute();}
+   *     {@code while (!operation.isSuccessful()) { ...}}
+   *
+ * + *

Please see the API + * specification of the export method for details. + * + *

The following undocumented behaviors with regard to {@code outputUrlPrefix} have been + * observed: + * + *

    + *
  • If outputUrlPrefix refers to a GCS bucket, exported data will be nested deeper in the + * bucket with a timestamped path. This is useful when periodical backups are desired + *
  • If outputUrlPrefix is a already a nested path in a GCS bucket, exported data will be put + * under this path. This means that a nested path is not reusable, since the export process + * by default would not overwrite existing files. + *
+ * + * @param outputUrlPrefix the full resource URL of the external storage location + * @param kinds the datastore 'kinds' to be exported + */ + public Export export(String outputUrlPrefix, Collection kinds) { + return new Export(new ExportRequest(outputUrlPrefix, kinds)); + } + + /** + * Returns a {@link Get} request that retrieves the details of an export or import {@link + * Operation}. + * + * @param operationName name of the {@code Operation} as returned by an export or import request + */ + public Get get(String operationName) { + return new Get(operationName); + } + + /** + * Returns a {@link ListOperations} request that retrieves all export or import {@link Operation + * operations} matching {@code filter}. + * + *

Sample usage: find all operations started after 2018-10-31 00:00:00 UTC and has stopped: + * + * 

+   *     {@code String filter = "metadata.common.startTime>\"2018-10-31T0:0:0Z\" AND done=true";}
+   *     {@code List operations = datastoreAdmin.list(filter);}
+   *
+ * + *

Please refer to {@link Operation} for how to reference operation properties. + */ + public ListOperations list(String filter) { + checkArgument(!Strings.isNullOrEmpty(filter), "Filter must not be null or empty."); + return new ListOperations(Optional.of(filter)); + } + + /** + * Returns a {@link ListOperations} request that retrieves all export or import {@link Operation * + * operations}. + */ + public ListOperations listAll() { + return new ListOperations(Optional.empty()); + } + + /** Builder for {@link DatastoreAdmin}. */ + public static class Builder extends AbstractGoogleJsonClient.Builder { + + private String projectId; + + public Builder( + HttpTransport httpTransport, + JsonFactory jsonFactory, + HttpRequestInitializer httpRequestInitializer) { + super(httpTransport, jsonFactory, ROOT_URL, SERVICE_PATH, httpRequestInitializer, false); + } + + @Override + public Builder setApplicationName(String applicationName) { + return (Builder) super.setApplicationName(applicationName); + } + + /** Sets the GCP project ID of the Cloud Datastore databases being managed. */ + public Builder setProjectId(String projectId) { + this.projectId = projectId; + return this; + } + + @Override + public DatastoreAdmin build() { + return new DatastoreAdmin(this); + } + } + + /** A request to export Cloud Datastore databases. */ + public class Export extends DatastoreAdminRequest { + + Export(ExportRequest exportRequest) { + super( + DatastoreAdmin.this, + "POST", + "projects/{projectId}:export", + exportRequest, + Operation.class); + set("projectId", projectId); + } + } + + /** A request to retrieve details of an export or import operation. */ + public class Get extends DatastoreAdminRequest { + + Get(String operationName) { + super(DatastoreAdmin.this, "GET", operationName, null, Operation.class); + } + } + + /** A request to retrieve all export or import operations matching a given filter. */ + public class ListOperations extends DatastoreAdminRequest { + + ListOperations(Optional filter) { + super( + DatastoreAdmin.this, + "GET", + "projects/{projectId}/operations", + null, + Operation.OperationList.class); + set("projectId", projectId); + filter.ifPresent(f -> set("filter", f)); + } + } + + /** Base class of all DatastoreAdmin requests. */ + abstract static class DatastoreAdminRequest extends AbstractGoogleJsonClientRequest { + /** + * @param client Google JSON client + * @param requestMethod HTTP Method + * @param uriTemplate URI template for the path relative to the base URL. If it starts with a + * "/" the base path from the base URL will be stripped out. The URI template can also be a + * full URL. URI template expansion is done using {@link + * com.google.api.client.http.UriTemplate#expand(String, String, Object, boolean)} + * @param jsonContent POJO that can be serialized into JSON content or {@code null} for none + * @param responseClass response class to parse into + */ + protected DatastoreAdminRequest( + DatastoreAdmin client, + String requestMethod, + String uriTemplate, + Object jsonContent, + Class responseClass) { + super(client, requestMethod, uriTemplate, jsonContent, responseClass); + } + } + + /** + * Model object that describes the JSON content in an export request. + * + *

Please note that some properties defined in the API are excluded, e.g., {@code databaseId} + * (not supported by Cloud Datastore) and labels (not used by Domain Registry). + */ + @SuppressWarnings("unused") + static class ExportRequest extends GenericJson { + @Key private final String outputUrlPrefix; + @Key private final EntityFilter entityFilter; + + ExportRequest(String outputUrlPrefix, Collection kinds) { + checkNotNull(outputUrlPrefix, "outputUrlPrefix"); + this.outputUrlPrefix = outputUrlPrefix; + this.entityFilter = new EntityFilter(kinds); + } + } +} diff --git a/java/google/registry/export/datastore/DatastoreAdminModule.java b/java/google/registry/export/datastore/DatastoreAdminModule.java new file mode 100644 index 000000000..322fb0e2f --- /dev/null +++ b/java/google/registry/export/datastore/DatastoreAdminModule.java @@ -0,0 +1,39 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export.datastore; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import dagger.Module; +import dagger.Provides; +import google.registry.config.CredentialModule; +import google.registry.config.RegistryConfig; +import javax.inject.Singleton; + +/** Dagger module that configures provision of {@link DatastoreAdmin}. */ +@Module +public abstract class DatastoreAdminModule { + + @Singleton + @Provides + static DatastoreAdmin provideDatastoreAdmin( + @CredentialModule.DefaultCredential GoogleCredential credential, + @RegistryConfig.Config("projectId") String projectId) { + return new DatastoreAdmin.Builder( + credential.getTransport(), credential.getJsonFactory(), credential) + .setApplicationName(projectId) + .setProjectId(projectId) + .build(); + } +} diff --git a/java/google/registry/export/datastore/EntityFilter.java b/java/google/registry/export/datastore/EntityFilter.java new file mode 100644 index 000000000..ce261b49d --- /dev/null +++ b/java/google/registry/export/datastore/EntityFilter.java @@ -0,0 +1,49 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export.datastore; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.api.client.json.GenericJson; +import com.google.api.client.util.Key; +import com.google.common.collect.ImmutableList; +import java.util.Collection; +import java.util.List; + +/** + * Model object that describes the Cloud Datastore 'kinds' to be exported or imported. The JSON form + * of this type is found in export/import requests and responses. + * + *

Please note that properties not used by Domain Registry are not included, e.g., {@code + * namespaceIds}. + */ +public class EntityFilter extends GenericJson { + + @Key private List kinds = ImmutableList.of(); + + /** For JSON deserialization. */ + public EntityFilter() {} + + EntityFilter(Collection kinds) { + checkNotNull(kinds, "kinds"); + checkArgument(!kinds.isEmpty(), "kinds must not be empty"); + this.kinds = ImmutableList.copyOf(kinds); + } + + List getKinds() { + return ImmutableList.copyOf(kinds); + } +} diff --git a/java/google/registry/export/datastore/Operation.java b/java/google/registry/export/datastore/Operation.java new file mode 100644 index 000000000..871cee365 --- /dev/null +++ b/java/google/registry/export/datastore/Operation.java @@ -0,0 +1,222 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export.datastore; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.api.client.json.GenericJson; +import com.google.api.client.util.Key; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import google.registry.export.datastore.DatastoreAdmin.Get; +import google.registry.util.Clock; +import java.util.List; +import java.util.Optional; +import javax.annotation.Nullable; +import org.joda.time.DateTime; +import org.joda.time.Duration; + +/** + * Model object that describes the details of an export or import operation in Cloud Datastore. + * + *

{@link Operation} instances are parsed from the JSON payload in Datastore response messages. + */ +public class Operation extends GenericJson { + + private static final String STATE_SUCCESS = "SUCCESSFUL"; + private static final String STATE_PROCESSING = "PROCESSING"; + + @Key private String name; + @Key private Metadata metadata; + @Key private boolean done; + + /** For JSON deserialization. */ + public Operation() {} + + /** Returns the name of this operation, which may be used in a {@link Get} request. */ + public String getName() { + checkState(name != null, "Name must not be null."); + return name; + } + + public boolean isDone() { + return done; + } + + private String getState() { + return getMetadata().getCommonMetadata().getState(); + } + + public boolean isSuccessful() { + return getState().equals(STATE_SUCCESS); + } + + public boolean isProcessing() { + return getState().equals(STATE_PROCESSING); + } + + /** + * Returns the elapsed time since starting if this operation is still running, or the total + * running time if this operation has completed. + */ + public Duration getRunningTime(Clock clock) { + return new Duration( + getStartTime(), getMetadata().getCommonMetadata().getEndTime().orElse(clock.nowUtc())); + } + + public DateTime getStartTime() { + return getMetadata().getCommonMetadata().getStartTime(); + } + + public ImmutableSet getKinds() { + return ImmutableSet.copyOf(getMetadata().getEntityFilter().getKinds()); + } + + /** + * Returns the URL to the GCS folder that holds the exported data. This folder is created by + * Datastore and is under the {@code outputUrlPrefix} set to {@linkplain + * DatastoreAdmin#export(String, List) the export request}. + */ + public String getExportFolderUrl() { + return getMetadata().getOutputUrlPrefix(); + } + + /** + * Returns the last segment of the {@linkplain #getExportFolderUrl() export folder URL} which can + * be used as unique identifier of this export operation. This is a better ID than the {@linkplain + * #getName() operation name}, which is opaque. + */ + public String getExportId() { + String exportFolderUrl = getExportFolderUrl(); + return exportFolderUrl.substring(exportFolderUrl.lastIndexOf('/') + 1); + } + + public String getProgress() { + StringBuilder result = new StringBuilder(); + Progress progress = getMetadata().getProgressBytes(); + if (progress != null) { + result.append( + String.format(" [%s/%s bytes]", progress.workCompleted, progress.workEstimated)); + } + progress = getMetadata().getProgressEntities(); + if (progress != null) { + result.append( + String.format(" [%s/%s entities]", progress.workCompleted, progress.workEstimated)); + } + if (result.length() == 0) { + return "Progress: N/A"; + } + return "Progress:" + result; + } + + private Metadata getMetadata() { + checkState(metadata != null, "Response metadata missing."); + return metadata; + } + + /** Models the common metadata properties of all operations. */ + public static class CommonMetadata extends GenericJson { + + @Key private String startTime; + @Key @Nullable private String endTime; + @Key private String operationType; + @Key private String state; + + public CommonMetadata() {} + + String getOperationType() { + checkState(!Strings.isNullOrEmpty(operationType), "operationType may not be null or empty"); + return operationType; + } + + String getState() { + checkState(!Strings.isNullOrEmpty(state), "state may not be null or empty"); + return state; + } + + DateTime getStartTime() { + checkState(startTime != null, "StartTime missing."); + return DateTime.parse(startTime); + } + + Optional getEndTime() { + return Optional.ofNullable(endTime).map(DateTime::parse); + } + } + + /** Models the metadata of a Cloud Datatore export or import operation. */ + public static class Metadata extends GenericJson { + @Key("common") + private CommonMetadata commonMetadata; + + @Key private Progress progressEntities; + @Key private Progress progressBytes; + @Key private EntityFilter entityFilter; + @Key private String outputUrlPrefix; + + public Metadata() {} + + CommonMetadata getCommonMetadata() { + checkState(commonMetadata != null, "CommonMetadata field is null."); + return commonMetadata; + } + + public Progress getProgressEntities() { + return progressEntities; + } + + public Progress getProgressBytes() { + return progressBytes; + } + + public EntityFilter getEntityFilter() { + return entityFilter; + } + + public String getOutputUrlPrefix() { + checkState(!Strings.isNullOrEmpty(outputUrlPrefix), "outputUrlPrefix"); + return outputUrlPrefix; + } + } + + /** Progress of an export or import operation. */ + public static class Progress extends GenericJson { + @Key private long workCompleted; + @Key private long workEstimated; + + public Progress() {} + + long getWorkCompleted() { + return workCompleted; + } + + public long getWorkEstimated() { + return workEstimated; + } + } + + /** List of {@link Operation Operations}. */ + public static class OperationList extends GenericJson { + @Key private List operations; + + /** For JSON deserialization. */ + public OperationList() {} + + ImmutableList toList() { + return ImmutableList.copyOf(operations); + } + } +} diff --git a/java/google/registry/flows/EppConsoleAction.java b/java/google/registry/flows/EppConsoleAction.java index 5d37c4089..23b4c4e24 100644 --- a/java/google/registry/flows/EppConsoleAction.java +++ b/java/google/registry/flows/EppConsoleAction.java @@ -15,10 +15,13 @@ package google.registry.flows; import com.google.appengine.api.users.UserService; +import google.registry.model.eppcommon.ProtocolDefinition; import google.registry.request.Action; import google.registry.request.Action.Method; +import google.registry.request.Parameter; import google.registry.request.Payload; import google.registry.request.auth.Auth; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; import javax.inject.Inject; import javax.servlet.http.HttpSession; @@ -34,13 +37,16 @@ public class EppConsoleAction implements Runnable { @Inject HttpSession session; @Inject EppRequestHandler eppRequestHandler; @Inject UserService userService; + @Inject AuthenticatedRegistrarAccessor registrarAccessor; + @Inject @Parameter("clientId") String clientId; @Inject EppConsoleAction() {} @Override public void run() { eppRequestHandler.executeEpp( - new HttpSessionMetadata(session), - GaeUserCredentials.forCurrentUser(userService), + new StatelessRequestSessionMetadata(clientId, + ProtocolDefinition.getVisibleServiceExtensionUris()), + new GaeUserCredentials(registrarAccessor), EppRequestSource.CONSOLE, false, // This endpoint is never a dry run. false, // This endpoint is never a superuser. diff --git a/java/google/registry/flows/EppController.java b/java/google/registry/flows/EppController.java index 7a43e3ff8..44f2ae054 100644 --- a/java/google/registry/flows/EppController.java +++ b/java/google/registry/flows/EppController.java @@ -17,8 +17,8 @@ package google.registry.flows; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.flogger.LazyArgs.lazy; import static com.google.common.io.BaseEncoding.base64; -import static google.registry.flows.EppXmlTransformer.unmarshal; import static google.registry.flows.FlowReporter.extractTlds; +import static google.registry.flows.FlowUtils.unmarshalEpp; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.annotations.VisibleForTesting; @@ -65,7 +65,7 @@ public final class EppController { try { EppInput eppInput; try { - eppInput = unmarshal(EppInput.class, inputXmlBytes); + eppInput = unmarshalEpp(EppInput.class, inputXmlBytes); } catch (EppException e) { // Log the unmarshalling error, with the raw bytes (in base64) to help with debugging. logger.atInfo().withCause(e).log( diff --git a/java/google/registry/flows/EppRequestHandler.java b/java/google/registry/flows/EppRequestHandler.java index 2d1f568fa..5b2785581 100644 --- a/java/google/registry/flows/EppRequestHandler.java +++ b/java/google/registry/flows/EppRequestHandler.java @@ -14,7 +14,7 @@ package google.registry.flows; -import static google.registry.flows.EppXmlTransformer.marshalWithLenientRetry; +import static google.registry.flows.FlowUtils.marshalWithLenientRetry; import static google.registry.model.eppoutput.Result.Code.SUCCESS_AND_CLOSE; import static google.registry.xml.XmlTransformer.prettyPrint; import static java.nio.charset.StandardCharsets.UTF_8; diff --git a/java/google/registry/flows/EppXmlTransformer.java b/java/google/registry/flows/EppXmlTransformer.java deleted file mode 100644 index 71838b3c8..000000000 --- a/java/google/registry/flows/EppXmlTransformer.java +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.flows; - -import static com.google.common.base.Preconditions.checkState; -import static google.registry.xml.ValidationMode.LENIENT; -import static google.registry.xml.ValidationMode.STRICT; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.flogger.FluentLogger; -import google.registry.flows.EppException.ParameterValueRangeErrorException; -import google.registry.flows.EppException.ParameterValueSyntaxErrorException; -import google.registry.flows.EppException.SyntaxErrorException; -import google.registry.flows.EppException.UnimplementedProtocolVersionException; -import google.registry.model.EppResourceUtils.InvalidRepoIdException; -import google.registry.model.ImmutableObject; -import google.registry.model.eppinput.EppInput; -import google.registry.model.eppinput.EppInput.WrongProtocolVersionException; -import google.registry.model.eppoutput.EppOutput; -import google.registry.model.host.InetAddressAdapter.IpVersionMismatchException; -import google.registry.model.translators.CurrencyUnitAdapter.UnknownCurrencyException; -import google.registry.xml.ValidationMode; -import google.registry.xml.XmlException; -import google.registry.xml.XmlTransformer; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.List; - -/** {@link XmlTransformer} for marshalling to and from the Epp model classes. */ -public class EppXmlTransformer { - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - - // Hardcoded XML schemas, ordered with respect to dependency. - private static final ImmutableList SCHEMAS = ImmutableList.of( - "eppcom.xsd", - "epp.xsd", - "contact.xsd", - "host.xsd", - "domain.xsd", - "rgp.xsd", - "secdns.xsd", - "fee06.xsd", - "fee11.xsd", - "fee12.xsd", - "metadata.xsd", - "mark.xsd", - "dsig.xsd", - "smd.xsd", - "launch.xsd", - "allocate.xsd", - "superuser.xsd", - "allocationToken-1.0.xsd"); - - private static final XmlTransformer INPUT_TRANSFORMER = - new XmlTransformer(SCHEMAS, EppInput.class); - - private static final XmlTransformer OUTPUT_TRANSFORMER = - new XmlTransformer(SCHEMAS, EppOutput.class); - - public static void validateOutput(String xml) throws XmlException { - OUTPUT_TRANSFORMER.validate(xml); - } - - /** - * Unmarshal bytes into Epp classes. - * - * @param clazz type to return, specified as a param to enforce typesafe generics - * @see TypeParameterUnusedInFormals - */ - public static T unmarshal(Class clazz, byte[] bytes) throws EppException { - try { - return INPUT_TRANSFORMER.unmarshal(clazz, new ByteArrayInputStream(bytes)); - } catch (XmlException e) { - // If this XmlException is wrapping a known type find it. If not, it's a syntax error. - List causalChain = Throwables.getCausalChain(e); - if (causalChain.stream().anyMatch(IpVersionMismatchException.class::isInstance)) { - throw new IpAddressVersionMismatchException(); - } - if (causalChain.stream().anyMatch(WrongProtocolVersionException.class::isInstance)) { - throw new UnimplementedProtocolVersionException(); - } - if (causalChain.stream().anyMatch(InvalidRepoIdException.class::isInstance)) { - throw new InvalidRepoIdEppException(); - } - if (causalChain.stream().anyMatch(UnknownCurrencyException.class::isInstance)) { - throw new UnknownCurrencyEppException(); - } - throw new GenericSyntaxErrorException(e.getMessage()); - } - } - - private static byte[] marshal( - XmlTransformer transformer, - ImmutableObject root, - ValidationMode validation) throws XmlException { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - transformer.marshal(root, byteArrayOutputStream, UTF_8, validation); - return byteArrayOutputStream.toByteArray(); - } - - public static byte[] marshal(EppOutput root, ValidationMode validation) throws XmlException { - return marshal(OUTPUT_TRANSFORMER, root, validation); - } - - public static byte[] marshalWithLenientRetry(EppOutput eppOutput) { - checkState(eppOutput != null); - // We need to marshal to a string instead of writing the response directly to the servlet's - // response writer, so that partial results don't get written on failure. - try { - return EppXmlTransformer.marshal(eppOutput, STRICT); - } catch (XmlException e) { - // We failed to marshal with validation. This is very bad, but we can potentially still send - // back slightly invalid xml, so try again without validation. - try { - byte[] lenient = EppXmlTransformer.marshal(eppOutput, LENIENT); - // Marshaling worked even though the results didn't validate against the schema. - logger.atSevere().withCause(e).log( - "Result marshaled but did not validate: %s", new String(lenient, UTF_8)); - return lenient; - } catch (XmlException e2) { - throw new RuntimeException(e2); // Failing to marshal at all is not recoverable. - } - } - } - - @VisibleForTesting - public static byte[] marshalInput(EppInput root, ValidationMode validation) throws XmlException { - return marshal(INPUT_TRANSFORMER, root, validation); - } - - @VisibleForTesting - public static void validateInput(String xml) throws XmlException { - INPUT_TRANSFORMER.validate(xml); - } - - /** IP address version mismatch. */ - public static class IpAddressVersionMismatchException extends ParameterValueRangeErrorException { - public IpAddressVersionMismatchException() { - super("IP adddress version mismatch"); - } - } - - /** Invalid format for repository id. */ - public static class InvalidRepoIdEppException extends ParameterValueSyntaxErrorException { - public InvalidRepoIdEppException() { - super("Invalid format for repository id"); - } - } - - /** Unknown currency. */ - static class UnknownCurrencyEppException extends ParameterValueRangeErrorException { - public UnknownCurrencyEppException() { - super("Unknown currency."); - } - } - - /** Generic syntax error that can be thrown by any flow. */ - static class GenericSyntaxErrorException extends SyntaxErrorException { - public GenericSyntaxErrorException(String message) { - super(message); - } - } -} diff --git a/java/google/registry/flows/FlowUtils.java b/java/google/registry/flows/FlowUtils.java index 238ef3951..767688e08 100644 --- a/java/google/registry/flows/FlowUtils.java +++ b/java/google/registry/flows/FlowUtils.java @@ -14,14 +14,32 @@ package google.registry.flows; +import static com.google.common.base.Preconditions.checkState; import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.xml.ValidationMode.LENIENT; +import static google.registry.xml.ValidationMode.STRICT; +import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.base.Throwables; +import com.google.common.flogger.FluentLogger; import google.registry.flows.EppException.CommandUseErrorException; +import google.registry.flows.EppException.ParameterValueRangeErrorException; +import google.registry.flows.EppException.SyntaxErrorException; +import google.registry.flows.EppException.UnimplementedProtocolVersionException; import google.registry.flows.custom.EntityChanges; +import google.registry.model.eppcommon.EppXmlTransformer; +import google.registry.model.eppinput.EppInput.WrongProtocolVersionException; +import google.registry.model.eppoutput.EppOutput; +import google.registry.model.host.InetAddressAdapter.IpVersionMismatchException; +import google.registry.model.translators.CurrencyUnitAdapter.UnknownCurrencyException; +import google.registry.xml.XmlException; +import java.util.List; /** Static utility functions for flows. */ public final class FlowUtils { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private FlowUtils() {} /** Validate that there is a logged in client. */ @@ -37,10 +55,75 @@ public final class FlowUtils { ofy().delete().keys(entityChanges.getDeletes()); } + /** + * Unmarshal bytes into Epp classes. Does the same as {@link EppXmlTransformer#unmarshal(Class, + * byte[])} but with exception-handling logic to throw {@link EppException} instead. + */ + public static T unmarshalEpp(Class clazz, byte[] bytes) throws EppException { + try { + return EppXmlTransformer.unmarshal(clazz, bytes); + } catch (XmlException e) { + // If this XmlException is wrapping a known type find it. If not, it's a syntax error. + List causalChain = Throwables.getCausalChain(e); + if (causalChain.stream().anyMatch(IpVersionMismatchException.class::isInstance)) { + throw new IpAddressVersionMismatchException(); + } + if (causalChain.stream().anyMatch(WrongProtocolVersionException.class::isInstance)) { + throw new UnimplementedProtocolVersionException(); + } + if (causalChain.stream().anyMatch(UnknownCurrencyException.class::isInstance)) { + throw new UnknownCurrencyEppException(); + } + throw new GenericXmlSyntaxErrorException(e.getMessage()); + } + } + + public static byte[] marshalWithLenientRetry(EppOutput eppOutput) { + checkState(eppOutput != null); + // We need to marshal to a string instead of writing the response directly to the servlet's + // response writer, so that partial results don't get written on failure. + try { + return EppXmlTransformer.marshal(eppOutput, STRICT); + } catch (XmlException e) { + // We failed to marshal with validation. This is very bad, but we can potentially still send + // back slightly invalid xml, so try again without validation. + try { + byte[] lenient = EppXmlTransformer.marshal(eppOutput, LENIENT); + // Marshaling worked even though the results didn't validate against the schema. + logger.atSevere().withCause(e).log( + "Result marshaled but did not validate: %s", new String(lenient, UTF_8)); + return lenient; + } catch (XmlException e2) { + throw new RuntimeException(e2); // Failing to marshal at all is not recoverable. + } + } + } + /** Registrar is not logged in. */ public static class NotLoggedInException extends CommandUseErrorException { public NotLoggedInException() { super("Registrar is not logged in."); } } + + /** IP address version mismatch. */ + public static class IpAddressVersionMismatchException extends ParameterValueRangeErrorException { + public IpAddressVersionMismatchException() { + super("IP adddress version mismatch"); + } + } + + /** Unknown currency. */ + static class UnknownCurrencyEppException extends ParameterValueRangeErrorException { + public UnknownCurrencyEppException() { + super("Unknown currency."); + } + } + + /** Generic XML syntax error that can be thrown by any flow. */ + public static class GenericXmlSyntaxErrorException extends SyntaxErrorException { + public GenericXmlSyntaxErrorException(String message) { + super(message); + } + } } diff --git a/java/google/registry/flows/GaeUserCredentials.java b/java/google/registry/flows/GaeUserCredentials.java index d4a211723..9e773e000 100644 --- a/java/google/registry/flows/GaeUserCredentials.java +++ b/java/google/registry/flows/GaeUserCredentials.java @@ -15,98 +15,40 @@ package google.registry.flows; import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.base.Strings.nullToEmpty; -import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; -import com.google.appengine.api.users.User; -import com.google.appengine.api.users.UserService; -import com.google.common.annotations.VisibleForTesting; import google.registry.flows.EppException.AuthenticationErrorException; import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import javax.annotation.Nullable; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.RegistrarAccessDeniedException; /** Credentials provided by {@link com.google.appengine.api.users.UserService}. */ public class GaeUserCredentials implements TransportCredentials { - private final User gaeUser; - private final Boolean isAdmin; + private final AuthenticatedRegistrarAccessor registrarAccessor; - /** - * Create an instance for the current user, as determined by {@code UserService}. - * - *

Note that the current user may be null (i.e. there is no logged in user). - */ - public static GaeUserCredentials forCurrentUser(UserService userService) { - User user = userService.getCurrentUser(); - return new GaeUserCredentials(user, user != null ? userService.isUserAdmin() : null); - } - - /** Create an instance that represents an explicit user (for testing purposes). */ - @VisibleForTesting - public static GaeUserCredentials forTestingUser(User gaeUser, Boolean isAdmin) { - checkArgumentNotNull(gaeUser); - checkArgumentNotNull(isAdmin); - return new GaeUserCredentials(gaeUser, isAdmin); - } - - /** Create an instance that represents a non-logged in user (for testing purposes). */ - @VisibleForTesting - public static GaeUserCredentials forLoggedOutUser() { - return new GaeUserCredentials(null, null); - } - - private GaeUserCredentials(@Nullable User gaeUser, @Nullable Boolean isAdmin) { - this.gaeUser = gaeUser; - this.isAdmin = isAdmin; - } - - @VisibleForTesting - User getUser() { - return gaeUser; + public GaeUserCredentials(AuthenticatedRegistrarAccessor registrarAccessor) { + this.registrarAccessor = registrarAccessor; } @Override public void validate(Registrar registrar, String ignoredPassword) throws AuthenticationErrorException { - if (gaeUser == null) { - throw new UserNotLoggedInException(); + try { + registrarAccessor.verifyAccess(registrar.getClientId()); + } catch (RegistrarAccessDeniedException e) { + throw new UserForbiddenException(e); } - // Allow admins to act as any registrar. - if (Boolean.TRUE.equals(isAdmin)) { - return; - } - // Check Registrar's contacts to see if any are associated with this gaeUserId. - final String gaeUserId = gaeUser.getUserId(); - for (RegistrarContact rc : registrar.getContacts()) { - if (gaeUserId.equals(rc.getGaeUserId())) { - return; - } - } - throw new BadGaeUserIdException(gaeUser); } @Override public String toString() { - return toStringHelper(getClass()) - .add("gaeUser", gaeUser) - .add("isAdmin", isAdmin) - .toString(); + return toStringHelper(getClass()).add("user", registrarAccessor.userIdForLogging()).toString(); } - /** User is not logged in as a GAE user. */ - public static class UserNotLoggedInException extends AuthenticationErrorException { - public UserNotLoggedInException() { - super("User is not logged in"); - } - } - - /** GAE user id is not allowed to login as requested registrar. */ - public static class BadGaeUserIdException extends AuthenticationErrorException { - public BadGaeUserIdException(User user) { - super( - "User id is not allowed to login as requested registrar: " - + (nullToEmpty(user.getEmail()))); + /** GAE User can't access the requested registrar. */ + public static class UserForbiddenException extends AuthenticationErrorException { + public UserForbiddenException(RegistrarAccessDeniedException e) { + super(e.getMessage()); } } } diff --git a/java/google/registry/flows/ResourceFlowUtils.java b/java/google/registry/flows/ResourceFlowUtils.java index a63501742..f94b30478 100644 --- a/java/google/registry/flows/ResourceFlowUtils.java +++ b/java/google/registry/flows/ResourceFlowUtils.java @@ -258,13 +258,17 @@ public final class ResourceFlowUtils { * Resolve a pending transfer by denying it. * *

This removes the {@link StatusValue#PENDING_TRANSFER} status, sets the {@link - * TransferStatus}, clears all the server-approve fields on the {@link TransferData}, and sets the - * expiration time of the last pending transfer to now. + * TransferStatus}, clears all the server-approve fields on the {@link TransferData}, sets the + * expiration time of the last pending transfer to now, sets the last EPP update time to now, and + * sets the last EPP update client id to the given client id. */ public static R denyPendingTransfer( - R resource, TransferStatus transferStatus, DateTime now) { + R resource, TransferStatus transferStatus, DateTime now, String lastEppUpdateClientId) { checkArgument(transferStatus.isDenied(), "Not a denial transfer status"); - return resolvePendingTransfer(resource, transferStatus, now).build(); + return resolvePendingTransfer(resource, transferStatus, now) + .setLastEppUpdateTime(now) + .setLastEppUpdateClientId(lastEppUpdateClientId) + .build(); } public static void verifyHasPendingTransfer( @@ -288,11 +292,8 @@ public final class ResourceFlowUtils { } public static R verifyExistence( - Class clazz, String targetId, R resource) throws ResourceDoesNotExistException { - if (resource == null) { - throw new ResourceDoesNotExistException(clazz, targetId); - } - return resource; + Class clazz, String targetId, Optional resource) throws ResourceDoesNotExistException { + return resource.orElseThrow(() -> new ResourceDoesNotExistException(clazz, targetId)); } public static void verifyResourceDoesNotExist( diff --git a/java/google/registry/flows/TlsCredentials.java b/java/google/registry/flows/TlsCredentials.java index f4119925c..455123e27 100644 --- a/java/google/registry/flows/TlsCredentials.java +++ b/java/google/registry/flows/TlsCredentials.java @@ -26,6 +26,7 @@ import com.google.common.net.HostAndPort; import com.google.common.net.InetAddresses; import dagger.Module; import dagger.Provides; +import google.registry.config.RegistryConfig.Config; import google.registry.flows.EppException.AuthenticationErrorException; import google.registry.model.registrar.Registrar; import google.registry.request.Header; @@ -54,14 +55,17 @@ public class TlsCredentials implements TransportCredentials { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final boolean requireSslCertificates; private final String clientCertificateHash; private final InetAddress clientInetAddr; @Inject @VisibleForTesting public TlsCredentials( + @Config("requireSslCertificates") boolean requireSslCertificates, @Header("X-SSL-Certificate") String clientCertificateHash, @Header("X-Forwarded-For") Optional clientAddress) { + this.requireSslCertificates = requireSslCertificates; this.clientCertificateHash = clientCertificateHash; this.clientInetAddr = clientAddress.isPresent() ? parseInetAddress(clientAddress.get()) : null; } @@ -112,13 +116,17 @@ public class TlsCredentials implements TransportCredentials { * @throws MissingRegistrarCertificateException if frontend didn't send certificate hash header * @throws BadRegistrarCertificateException if registrar requires certificate and it didn't match */ - private void validateCertificate(Registrar registrar) throws AuthenticationErrorException { + @VisibleForTesting + void validateCertificate(Registrar registrar) throws AuthenticationErrorException { if (isNullOrEmpty(registrar.getClientCertificateHash()) && isNullOrEmpty(registrar.getFailoverClientCertificateHash())) { - logger.atInfo().log( - "Skipping SSL certificate check because %s doesn't have any certificate hashes on file", - registrar.getClientId()); - return; + if (requireSslCertificates) { + throw new RegistrarCertificateNotConfiguredException(); + } else { + // If the environment is configured to allow missing SSL certificate hashes and this hash is + // missing, then bypass the certificate hash checks. + return; + } } if (isNullOrEmpty(clientCertificateHash)) { logger.atInfo().log("Request did not include X-SSL-Certificate"); @@ -165,6 +173,14 @@ public class TlsCredentials implements TransportCredentials { } } + /** Registrar certificate is not configured. */ + public static class RegistrarCertificateNotConfiguredException + extends AuthenticationErrorException { + public RegistrarCertificateNotConfiguredException() { + super("Registrar certificate is not configured"); + } + } + /** Registrar IP address is not in stored whitelist. */ public static class BadRegistrarIpAddressException extends AuthenticationErrorException { public BadRegistrarIpAddressException() { diff --git a/java/google/registry/flows/contact/ContactTransferCancelFlow.java b/java/google/registry/flows/contact/ContactTransferCancelFlow.java index e265d6bc3..3dbec6b30 100644 --- a/java/google/registry/flows/contact/ContactTransferCancelFlow.java +++ b/java/google/registry/flows/contact/ContactTransferCancelFlow.java @@ -80,7 +80,7 @@ public final class ContactTransferCancelFlow implements TransactionalFlow { verifyHasPendingTransfer(existingContact); verifyTransferInitiator(clientId, existingContact); ContactResource newContact = - denyPendingTransfer(existingContact, TransferStatus.CLIENT_CANCELLED, now); + denyPendingTransfer(existingContact, TransferStatus.CLIENT_CANCELLED, now, clientId); HistoryEntry historyEntry = historyBuilder .setType(HistoryEntry.Type.CONTACT_TRANSFER_CANCEL) .setModificationTime(now) diff --git a/java/google/registry/flows/contact/ContactTransferRejectFlow.java b/java/google/registry/flows/contact/ContactTransferRejectFlow.java index 504e08d9b..ccded2b25 100644 --- a/java/google/registry/flows/contact/ContactTransferRejectFlow.java +++ b/java/google/registry/flows/contact/ContactTransferRejectFlow.java @@ -78,7 +78,7 @@ public final class ContactTransferRejectFlow implements TransactionalFlow { verifyHasPendingTransfer(existingContact); verifyResourceOwnership(clientId, existingContact); ContactResource newContact = - denyPendingTransfer(existingContact, TransferStatus.CLIENT_REJECTED, now); + denyPendingTransfer(existingContact, TransferStatus.CLIENT_REJECTED, now, clientId); HistoryEntry historyEntry = historyBuilder .setType(HistoryEntry.Type.CONTACT_TRANSFER_REJECT) .setModificationTime(now) diff --git a/java/google/registry/flows/domain/DomainAllocateFlow.java b/java/google/registry/flows/domain/DomainAllocateFlow.java index 3812805d6..bac773ea2 100644 --- a/java/google/registry/flows/domain/DomainAllocateFlow.java +++ b/java/google/registry/flows/domain/DomainAllocateFlow.java @@ -98,6 +98,8 @@ import org.joda.time.Duration; /** * An EPP flow that allocates a new domain resource from a domain application. * + *

Note that this flow is only run by superusers. + * * @error {@link google.registry.flows.exceptions.ResourceAlreadyExistsException} * @error {@link DomainAllocateFlow.HasFinalStatusException} * @error {@link DomainAllocateFlow.MissingApplicationException} @@ -222,10 +224,9 @@ public class DomainAllocateFlow implements TransactionalFlow { private DomainApplication loadAndValidateApplication( String applicationRoid, DateTime now) throws EppException { - DomainApplication application = loadDomainApplication(applicationRoid, now); - if (application == null) { - throw new MissingApplicationException(applicationRoid); - } + DomainApplication application = + loadDomainApplication(applicationRoid, now) + .orElseThrow(() -> new MissingApplicationException(applicationRoid)); if (application.getApplicationStatus().isFinalStatus()) { throw new HasFinalStatusException(); } diff --git a/java/google/registry/flows/domain/DomainApplicationCreateFlow.java b/java/google/registry/flows/domain/DomainApplicationCreateFlow.java index 7cb9a8bf0..770099915 100644 --- a/java/google/registry/flows/domain/DomainApplicationCreateFlow.java +++ b/java/google/registry/flows/domain/DomainApplicationCreateFlow.java @@ -147,7 +147,7 @@ import org.joda.time.DateTime; * @error {@link DomainFlowUtils.NotAuthorizedForTldException} * @error {@link DomainFlowUtils.PremiumNameBlockedException} * @error {@link DomainFlowUtils.RegistrantNotAllowedException} - * @error {@link DomainFlowUtils.RegistrarMustBeActiveToCreateDomainsException} + * @error {@link DomainFlowUtils.RegistrarMustBeActiveForThisOperationException} * @error {@link DomainFlowTmchUtils.SignedMarksMustBeEncodedException} * @error {@link DomainFlowTmchUtils.SignedMarkCertificateExpiredException} * @error {@link DomainFlowTmchUtils.SignedMarkCertificateInvalidException} diff --git a/java/google/registry/flows/domain/DomainApplicationInfoFlow.java b/java/google/registry/flows/domain/DomainApplicationInfoFlow.java index 841a0156e..4a89ecb7a 100644 --- a/java/google/registry/flows/domain/DomainApplicationInfoFlow.java +++ b/java/google/registry/flows/domain/DomainApplicationInfoFlow.java @@ -15,7 +15,7 @@ package google.registry.flows.domain; import static com.google.common.collect.Sets.union; -import static google.registry.flows.EppXmlTransformer.unmarshal; +import static google.registry.flows.FlowUtils.unmarshalEpp; import static google.registry.flows.FlowUtils.validateClientIsLoggedIn; import static google.registry.flows.ResourceFlowUtils.verifyExistence; import static google.registry.flows.ResourceFlowUtils.verifyOptionalAuthInfo; @@ -23,10 +23,10 @@ import static google.registry.flows.ResourceFlowUtils.verifyResourceOwnership; import static google.registry.flows.domain.DomainFlowUtils.addSecDnsExtensionIfPresent; import static google.registry.flows.domain.DomainFlowUtils.loadForeignKeyedDesignatedContacts; import static google.registry.flows.domain.DomainFlowUtils.verifyApplicationDomainMatchesTargetId; +import static google.registry.model.EppResourceUtils.loadDomainApplication; import static google.registry.model.ofy.ObjectifyService.ofy; import com.google.common.collect.ImmutableList; -import com.googlecode.objectify.Key; import google.registry.flows.EppException; import google.registry.flows.EppException.ParameterValuePolicyErrorException; import google.registry.flows.EppException.RequiredParameterMissingException; @@ -89,13 +89,10 @@ public final class DomainApplicationInfoFlow implements Flow { throw new MissingApplicationIdException(); } DomainApplication application = - ofy().load().key(Key.create(DomainApplication.class, applicationId)).now(); - verifyExistence( - DomainApplication.class, - applicationId, - application != null && clock.nowUtc().isBefore(application.getDeletionTime()) - ? application - : null); + verifyExistence( + DomainApplication.class, + applicationId, + loadDomainApplication(applicationId, clock.nowUtc())); verifyApplicationDomainMatchesTargetId(application, targetId); verifyOptionalAuthInfo(authInfo, application); LaunchInfoExtension launchInfo = eppInput.getSingleExtension(LaunchInfoExtension.class).get(); @@ -136,7 +133,7 @@ public final class DomainApplicationInfoFlow implements Flow { if (Boolean.TRUE.equals(launchInfo.getIncludeMark())) { // Default to false. for (EncodedSignedMark encodedMark : application.getEncodedSignedMarks()) { try { - marksBuilder.add(unmarshal(SignedMark.class, encodedMark.getBytes()).getMark()); + marksBuilder.add(unmarshalEpp(SignedMark.class, encodedMark.getBytes()).getMark()); } catch (EppException e) { // This is a serious error; don't let the benign EppException propagate. throw new IllegalStateException("Could not decode a stored encoded signed mark", e); diff --git a/java/google/registry/flows/domain/DomainCreateFlow.java b/java/google/registry/flows/domain/DomainCreateFlow.java index 09dce93b1..3a566e81a 100644 --- a/java/google/registry/flows/domain/DomainCreateFlow.java +++ b/java/google/registry/flows/domain/DomainCreateFlow.java @@ -179,7 +179,7 @@ import org.joda.time.Duration; * @error {@link DomainFlowUtils.NameserversNotSpecifiedForTldWithNameserverWhitelistException} * @error {@link DomainFlowUtils.PremiumNameBlockedException} * @error {@link DomainFlowUtils.RegistrantNotAllowedException} - * @error {@link DomainFlowUtils.RegistrarMustBeActiveToCreateDomainsException} + * @error {@link DomainFlowUtils.RegistrarMustBeActiveForThisOperationException} * @error {@link DomainFlowUtils.TldDoesNotExistException} * @error {@link DomainFlowUtils.TooManyDsRecordsException} * @error {@link DomainFlowUtils.TooManyNameserversException} diff --git a/java/google/registry/flows/domain/DomainDeleteFlow.java b/java/google/registry/flows/domain/DomainDeleteFlow.java index 29056bf77..3e1ce84e1 100644 --- a/java/google/registry/flows/domain/DomainDeleteFlow.java +++ b/java/google/registry/flows/domain/DomainDeleteFlow.java @@ -152,10 +152,12 @@ public final class DomainDeleteFlow implements TransactionalFlow { Builder builder; if (existingDomain.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) { builder = - denyPendingTransfer(existingDomain, TransferStatus.SERVER_CANCELLED, now).asBuilder(); + denyPendingTransfer(existingDomain, TransferStatus.SERVER_CANCELLED, now, clientId) + .asBuilder(); } else { builder = existingDomain.asBuilder(); } + builder.setLastEppUpdateTime(now).setLastEppUpdateClientId(clientId); Duration redemptionGracePeriodLength = registry.getRedemptionGracePeriodLength(); Duration pendingDeleteLength = registry.getPendingDeleteLength(); Optional domainDeleteSuperuserExtension = diff --git a/java/google/registry/flows/domain/DomainFlowTmchUtils.java b/java/google/registry/flows/domain/DomainFlowTmchUtils.java index 4ba9e8e4f..765e363fa 100644 --- a/java/google/registry/flows/domain/DomainFlowTmchUtils.java +++ b/java/google/registry/flows/domain/DomainFlowTmchUtils.java @@ -15,7 +15,7 @@ package google.registry.flows.domain; import static com.google.common.collect.Iterables.concat; -import static google.registry.flows.EppXmlTransformer.unmarshal; +import static google.registry.flows.FlowUtils.unmarshalEpp; import com.google.common.collect.ImmutableList; import google.registry.flows.EppException; @@ -90,7 +90,7 @@ public final class DomainFlowTmchUtils { SignedMark signedMark; try { - signedMark = unmarshal(SignedMark.class, signedMarkData); + signedMark = unmarshalEpp(SignedMark.class, signedMarkData); } catch (EppException e) { throw new SignedMarkParsingErrorException(); } diff --git a/java/google/registry/flows/domain/DomainFlowUtils.java b/java/google/registry/flows/domain/DomainFlowUtils.java index 57ae9c3c8..8223441ed 100644 --- a/java/google/registry/flows/domain/DomainFlowUtils.java +++ b/java/google/registry/flows/domain/DomainFlowUtils.java @@ -520,7 +520,7 @@ public class DomainFlowUtils { * Fills in a builder with the data needed for an autorenew billing event for this domain. This * does not copy over the id of the current autorenew billing event. */ - static BillingEvent.Recurring.Builder newAutorenewBillingEvent(DomainResource domain) { + public static BillingEvent.Recurring.Builder newAutorenewBillingEvent(DomainResource domain) { return new BillingEvent.Recurring.Builder() .setReason(Reason.RENEW) .setFlags(ImmutableSet.of(Flag.AUTO_RENEW)) @@ -533,7 +533,7 @@ public class DomainFlowUtils { * Fills in a builder with the data needed for an autorenew poll message for this domain. This * does not copy over the id of the current autorenew poll message. */ - static PollMessage.Autorenew.Builder newAutorenewPollMessage(DomainResource domain) { + public static PollMessage.Autorenew.Builder newAutorenewPollMessage(DomainResource domain) { return new PollMessage.Autorenew.Builder() .setTargetId(domain.getFullyQualifiedDomainName()) .setClientId(domain.getCurrentSponsorClientId()) @@ -542,12 +542,14 @@ public class DomainFlowUtils { } /** - * Re-saves the current autorenew billing event and poll message with a new end time. This may end - * up deleting the poll message (if closing the message interval) or recreating it (if opening the - * message interval). + * Re-saves the current autorenew billing event and poll message with a new end time. + * + *

This may end up deleting the poll message (if closing the message interval) or recreating it + * (if opening the message interval). This may cause an autorenew billing event to have an end + * time earlier than its event time (i.e. if it's being ended before it was ever triggered). */ @SuppressWarnings("unchecked") - static void updateAutorenewRecurrenceEndTime(DomainResource domain, DateTime newEndTime) { + public static void updateAutorenewRecurrenceEndTime(DomainResource domain, DateTime newEndTime) { Optional autorenewPollMessage = Optional.ofNullable(ofy().load().key(domain.getAutorenewPollMessage()).now()); @@ -892,13 +894,14 @@ public class DomainFlowUtils { /** * Check that the registrar with the given client ID is active. * - *

Non-active registrars are not allowed to create domain applications or domain resources. + *

Non-active registrars are not allowed to run operations that cost money, like domain creates + * or renews. */ static void verifyRegistrarIsActive(String clientId) - throws RegistrarMustBeActiveToCreateDomainsException { + throws RegistrarMustBeActiveForThisOperationException { Registrar registrar = Registrar.loadByClientIdCached(clientId).get(); if (registrar.getState() != State.ACTIVE) { - throw new RegistrarMustBeActiveToCreateDomainsException(); + throw new RegistrarMustBeActiveForThisOperationException(); } } @@ -1606,10 +1609,10 @@ public class DomainFlowUtils { } } - /** Registrar must be active in order to create domains or applications. */ - static class RegistrarMustBeActiveToCreateDomainsException extends AuthorizationErrorException { - public RegistrarMustBeActiveToCreateDomainsException() { - super("Registrar must be active in order to create domains or applications"); + /** Registrar must be active in order to perform this operation. */ + static class RegistrarMustBeActiveForThisOperationException extends AuthorizationErrorException { + public RegistrarMustBeActiveForThisOperationException() { + super("Registrar must be active in order to perform this operation"); } } } diff --git a/java/google/registry/flows/domain/DomainRenewFlow.java b/java/google/registry/flows/domain/DomainRenewFlow.java index e4bce5c94..3fadf8026 100644 --- a/java/google/registry/flows/domain/DomainRenewFlow.java +++ b/java/google/registry/flows/domain/DomainRenewFlow.java @@ -26,6 +26,7 @@ import static google.registry.flows.domain.DomainFlowUtils.newAutorenewPollMessa import static google.registry.flows.domain.DomainFlowUtils.updateAutorenewRecurrenceEndTime; import static google.registry.flows.domain.DomainFlowUtils.validateFeeChallenge; import static google.registry.flows.domain.DomainFlowUtils.validateRegistrationPeriod; +import static google.registry.flows.domain.DomainFlowUtils.verifyRegistrarIsActive; import static google.registry.flows.domain.DomainFlowUtils.verifyUnitIsYears; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.util.DateTimeUtils.leapSafeAddYears; @@ -101,6 +102,7 @@ import org.joda.time.Duration; * @error {@link DomainFlowUtils.FeesMismatchException} * @error {@link DomainFlowUtils.FeesRequiredForPremiumNameException} * @error {@link DomainFlowUtils.NotAuthorizedForTldException} + * @error {@link DomainFlowUtils.RegistrarMustBeActiveForThisOperationException} * @error {@link DomainFlowUtils.UnsupportedFeeAttributeException} * @error {@link DomainRenewFlow.IncorrectCurrentExpirationDateException} */ @@ -133,6 +135,7 @@ public final class DomainRenewFlow implements TransactionalFlow { flowCustomLogic.beforeValidation(); extensionManager.validate(); validateClientIsLoggedIn(clientId); + verifyRegistrarIsActive(clientId); DateTime now = ofy().getTransactionTime(); Renew command = (Renew) resourceCommand; // Loads the target resource if it exists @@ -172,12 +175,17 @@ public final class DomainRenewFlow implements TransactionalFlow { .build(); // End the old autorenew billing event and poll message now. This may delete the poll message. updateAutorenewRecurrenceEndTime(existingDomain, now); - DomainResource newDomain = existingDomain.asBuilder() - .setRegistrationExpirationTime(newExpirationTime) - .setAutorenewBillingEvent(Key.create(newAutorenewEvent)) - .setAutorenewPollMessage(Key.create(newAutorenewPollMessage)) - .addGracePeriod(GracePeriod.forBillingEvent(GracePeriodStatus.RENEW, explicitRenewEvent)) - .build(); + DomainResource newDomain = + existingDomain + .asBuilder() + .setLastEppUpdateTime(now) + .setLastEppUpdateClientId(clientId) + .setRegistrationExpirationTime(newExpirationTime) + .setAutorenewBillingEvent(Key.create(newAutorenewEvent)) + .setAutorenewPollMessage(Key.create(newAutorenewPollMessage)) + .addGracePeriod( + GracePeriod.forBillingEvent(GracePeriodStatus.RENEW, explicitRenewEvent)) + .build(); EntityChanges entityChanges = flowCustomLogic.beforeSave( BeforeSaveParameters.newBuilder() @@ -218,7 +226,7 @@ public final class DomainRenewFlow implements TransactionalFlow { .setType(HistoryEntry.Type.DOMAIN_RENEW) .setPeriod(period) .setModificationTime(now) - .setParent(Key.create(existingDomain)) + .setParent(existingDomain) .setDomainTransactionRecords( ImmutableSet.of( DomainTransactionRecord.create( diff --git a/java/google/registry/flows/domain/DomainRestoreRequestFlow.java b/java/google/registry/flows/domain/DomainRestoreRequestFlow.java index f109feaa8..d09613e19 100644 --- a/java/google/registry/flows/domain/DomainRestoreRequestFlow.java +++ b/java/google/registry/flows/domain/DomainRestoreRequestFlow.java @@ -25,6 +25,7 @@ import static google.registry.flows.domain.DomainFlowUtils.newAutorenewPollMessa import static google.registry.flows.domain.DomainFlowUtils.validateFeeChallenge; import static google.registry.flows.domain.DomainFlowUtils.verifyNotReserved; import static google.registry.flows.domain.DomainFlowUtils.verifyPremiumNameIsNotBlocked; +import static google.registry.flows.domain.DomainFlowUtils.verifyRegistrarIsActive; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.util.DateTimeUtils.END_OF_TIME; @@ -101,6 +102,7 @@ import org.joda.time.DateTime; * @error {@link DomainFlowUtils.FeesRequiredForPremiumNameException} * @error {@link DomainFlowUtils.NotAuthorizedForTldException} * @error {@link DomainFlowUtils.PremiumNameBlockedException} + * @error {@link DomainFlowUtils.RegistrarMustBeActiveForThisOperationException} * @error {@link DomainFlowUtils.UnsupportedFeeAttributeException} * @error {@link DomainRestoreRequestFlow.DomainNotEligibleForRestoreException} * @error {@link DomainRestoreRequestFlow.RestoreCommandIncludesChangesException} @@ -129,6 +131,7 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow { RgpUpdateExtension.class); extensionManager.validate(); validateClientIsLoggedIn(clientId); + verifyRegistrarIsActive(clientId); Update command = (Update) resourceCommand; DateTime now = ofy().getTransactionTime(); DomainResource existingDomain = loadAndVerifyExistence(DomainResource.class, targetId, now); @@ -158,7 +161,8 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow { .setParent(historyEntry) .build(); DomainResource newDomain = - performRestore(existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage); + performRestore( + existingDomain, newExpirationTime, autorenewEvent, autorenewPollMessage, now, clientId); updateForeignKeyIndexDeletionTime(newDomain); entitiesToSave.add(newDomain, historyEntry, autorenewEvent, autorenewPollMessage); ofy().save().entities(entitiesToSave.build()); @@ -224,8 +228,11 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow { DomainResource existingDomain, DateTime newExpirationTime, BillingEvent.Recurring autorenewEvent, - PollMessage.Autorenew autorenewPollMessage) { - return existingDomain.asBuilder() + PollMessage.Autorenew autorenewPollMessage, + DateTime now, + String clientId) { + return existingDomain + .asBuilder() .setRegistrationExpirationTime(newExpirationTime) .setDeletionTime(END_OF_TIME) .setStatusValues(null) @@ -233,6 +240,8 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow { .setDeletePollMessage(null) .setAutorenewBillingEvent(Key.create(autorenewEvent)) .setAutorenewPollMessage(Key.create(autorenewPollMessage)) + .setLastEppUpdateTime(now) + .setLastEppUpdateClientId(clientId) .build(); } diff --git a/java/google/registry/flows/domain/DomainTransferApproveFlow.java b/java/google/registry/flows/domain/DomainTransferApproveFlow.java index 3f5d5da44..d4ae38f32 100644 --- a/java/google/registry/flows/domain/DomainTransferApproveFlow.java +++ b/java/google/registry/flows/domain/DomainTransferApproveFlow.java @@ -181,7 +181,9 @@ public final class DomainTransferApproveFlow implements TransactionalFlow { // Update the transferredRegistrationExpirationTime here since approvePendingTransfer() // doesn't know what to set it to and leaves it null. .setTransferData( - partiallyApprovedDomain.getTransferData().asBuilder() + partiallyApprovedDomain + .getTransferData() + .asBuilder() .setTransferredRegistrationExpirationTime(newExpirationTime) .build()) .setRegistrationExpirationTime(newExpirationTime) @@ -193,6 +195,8 @@ public final class DomainTransferApproveFlow implements TransactionalFlow { ? ImmutableSet.of( GracePeriod.forBillingEvent(GracePeriodStatus.TRANSFER, billingEvent.get())) : ImmutableSet.of()) + .setLastEppUpdateTime(now) + .setLastEppUpdateClientId(clientId) .build(); // Create a poll message for the gaining client. PollMessage gainingClientPollMessage = createGainingTransferPollMessage( diff --git a/java/google/registry/flows/domain/DomainTransferCancelFlow.java b/java/google/registry/flows/domain/DomainTransferCancelFlow.java index a285b00ad..19b3b6c41 100644 --- a/java/google/registry/flows/domain/DomainTransferCancelFlow.java +++ b/java/google/registry/flows/domain/DomainTransferCancelFlow.java @@ -98,7 +98,7 @@ public final class DomainTransferCancelFlow implements TransactionalFlow { Registry registry = Registry.get(existingDomain.getTld()); HistoryEntry historyEntry = buildHistoryEntry(existingDomain, registry, now); DomainResource newDomain = - denyPendingTransfer(existingDomain, TransferStatus.CLIENT_CANCELLED, now); + denyPendingTransfer(existingDomain, TransferStatus.CLIENT_CANCELLED, now, clientId); ofy().save().entities( newDomain, historyEntry, diff --git a/java/google/registry/flows/domain/DomainTransferRejectFlow.java b/java/google/registry/flows/domain/DomainTransferRejectFlow.java index fdf561be6..a74e85a97 100644 --- a/java/google/registry/flows/domain/DomainTransferRejectFlow.java +++ b/java/google/registry/flows/domain/DomainTransferRejectFlow.java @@ -100,7 +100,7 @@ public final class DomainTransferRejectFlow implements TransactionalFlow { checkAllowedAccessToTld(clientId, existingDomain.getTld()); } DomainResource newDomain = - denyPendingTransfer(existingDomain, TransferStatus.CLIENT_REJECTED, now); + denyPendingTransfer(existingDomain, TransferStatus.CLIENT_REJECTED, now, clientId); ofy().save().entities( newDomain, historyEntry, diff --git a/java/google/registry/flows/domain/DomainTransferRequestFlow.java b/java/google/registry/flows/domain/DomainTransferRequestFlow.java index ddff94e53..fb10545dd 100644 --- a/java/google/registry/flows/domain/DomainTransferRequestFlow.java +++ b/java/google/registry/flows/domain/DomainTransferRequestFlow.java @@ -23,6 +23,7 @@ import static google.registry.flows.domain.DomainFlowUtils.checkAllowedAccessToT import static google.registry.flows.domain.DomainFlowUtils.updateAutorenewRecurrenceEndTime; import static google.registry.flows.domain.DomainFlowUtils.validateFeeChallenge; import static google.registry.flows.domain.DomainFlowUtils.verifyPremiumNameIsNotBlocked; +import static google.registry.flows.domain.DomainFlowUtils.verifyRegistrarIsActive; import static google.registry.flows.domain.DomainFlowUtils.verifyUnitIsYears; import static google.registry.flows.domain.DomainTransferUtils.createLosingTransferPollMessage; import static google.registry.flows.domain.DomainTransferUtils.createPendingTransferData; @@ -107,6 +108,7 @@ import org.joda.time.DateTime; * @error {@link DomainFlowUtils.FeesRequiredForPremiumNameException} * @error {@link DomainFlowUtils.NotAuthorizedForTldException} * @error {@link DomainFlowUtils.PremiumNameBlockedException} + * @error {@link DomainFlowUtils.RegistrarMustBeActiveForThisOperationException} * @error {@link DomainFlowUtils.UnsupportedFeeAttributeException} */ @ReportingSpec(ActivityReportField.DOMAIN_TRANSFER_REQUEST) @@ -139,6 +141,7 @@ public final class DomainTransferRequestFlow implements TransactionalFlow { MetadataExtension.class); extensionManager.validate(); validateClientIsLoggedIn(gainingClientId); + verifyRegistrarIsActive(gainingClientId); DateTime now = ofy().getTransactionTime(); DomainResource existingDomain = loadAndVerifyExistence(DomainResource.class, targetId, now); Optional superuserExtension = @@ -222,10 +225,14 @@ public final class DomainTransferRequestFlow implements TransactionalFlow { // cloneProjectedAtTime() will replace these old autorenew entities with the server approve ones // that we've created in this flow and stored in pendingTransferData. updateAutorenewRecurrenceEndTime(existingDomain, automaticTransferTime); - DomainResource newDomain = existingDomain.asBuilder() - .setTransferData(pendingTransferData) - .addStatusValue(StatusValue.PENDING_TRANSFER) - .build(); + DomainResource newDomain = + existingDomain + .asBuilder() + .setTransferData(pendingTransferData) + .addStatusValue(StatusValue.PENDING_TRANSFER) + .setLastEppUpdateTime(now) + .setLastEppUpdateClientId(gainingClientId) + .build(); asyncFlowEnqueuer.enqueueAsyncResave(newDomain, now, automaticTransferTime); ofy().save() .entities(new ImmutableSet.Builder<>() diff --git a/java/google/registry/flows/host/HostCreateFlow.java b/java/google/registry/flows/host/HostCreateFlow.java index 2ffbe8ccf..63aa8c0a0 100644 --- a/java/google/registry/flows/host/HostCreateFlow.java +++ b/java/google/registry/flows/host/HostCreateFlow.java @@ -64,7 +64,7 @@ import org.joda.time.DateTime; * hosts cannot have any. This flow allows creating a host name, and if necessary enqueues tasks to * update DNS. * - * @error {@link google.registry.flows.EppXmlTransformer.IpAddressVersionMismatchException} + * @error {@link google.registry.flows.FlowUtils.IpAddressVersionMismatchException} * @error {@link google.registry.flows.exceptions.ResourceAlreadyExistsException} * @error {@link HostFlowUtils.HostNameTooLongException} * @error {@link HostFlowUtils.HostNameTooShallowException} @@ -87,8 +87,13 @@ public final class HostCreateFlow implements TransactionalFlow { @Inject HistoryEntry.Builder historyBuilder; @Inject DnsQueue dnsQueue; @Inject EppResponse.Builder responseBuilder; - @Inject @Config("contactAndHostRoidSuffix") String roidSuffix; - @Inject HostCreateFlow() {} + + @Inject + @Config("contactAndHostRoidSuffix") + String roidSuffix; + + @Inject + HostCreateFlow() {} @Override public final EppResponse run() throws EppException { @@ -126,17 +131,21 @@ public final class HostCreateFlow implements TransactionalFlow { .setType(HistoryEntry.Type.HOST_CREATE) .setModificationTime(now) .setParent(Key.create(newHost)); - ImmutableSet entitiesToSave = ImmutableSet.of( - newHost, - historyBuilder.build(), - ForeignKeyIndex.create(newHost, newHost.getDeletionTime()), - EppResourceIndex.create(Key.create(newHost))); + ImmutableSet entitiesToSave = + ImmutableSet.of( + newHost, + historyBuilder.build(), + ForeignKeyIndex.create(newHost, newHost.getDeletionTime()), + EppResourceIndex.create(Key.create(newHost))); if (superordinateDomain.isPresent()) { - entitiesToSave = union( - entitiesToSave, - superordinateDomain.get().asBuilder() - .addSubordinateHost(command.getFullyQualifiedHostName()) - .build()); + entitiesToSave = + union( + entitiesToSave, + superordinateDomain + .get() + .asBuilder() + .addSubordinateHost(command.getFullyQualifiedHostName()) + .build()); // Only update DNS if this is a subordinate host. External hosts have no glue to write, so // they are only written as NS records from the referencing domain. dnsQueue.addHostRefreshTask(targetId); diff --git a/java/google/registry/flows/host/HostFlowUtils.java b/java/google/registry/flows/host/HostFlowUtils.java index 41da5bea9..704eaa397 100644 --- a/java/google/registry/flows/host/HostFlowUtils.java +++ b/java/google/registry/flows/host/HostFlowUtils.java @@ -87,16 +87,15 @@ public class HostFlowUtils { } // This is a subordinate host String domainName = - hostName - .parts() - .stream() + hostName.parts().stream() .skip(hostName.parts().size() - (tld.get().parts().size() + 1)) .collect(joining(".")); - DomainResource superordinateDomain = loadByForeignKey(DomainResource.class, domainName, now); - if (superordinateDomain == null || !isActive(superordinateDomain, now)) { + Optional superordinateDomain = + loadByForeignKey(DomainResource.class, domainName, now); + if (!superordinateDomain.isPresent() || !isActive(superordinateDomain.get(), now)) { throw new SuperordinateDomainDoesNotExistException(domainName); } - return Optional.of(superordinateDomain); + return superordinateDomain; } /** Superordinate domain for this hostname does not exist. */ diff --git a/java/google/registry/flows/session/LoginFlow.java b/java/google/registry/flows/session/LoginFlow.java index bebd4603c..78b949ad6 100644 --- a/java/google/registry/flows/session/LoginFlow.java +++ b/java/google/registry/flows/session/LoginFlow.java @@ -51,8 +51,7 @@ import javax.inject.Inject; * @error {@link google.registry.flows.EppException.UnimplementedExtensionException} * @error {@link google.registry.flows.EppException.UnimplementedObjectServiceException} * @error {@link google.registry.flows.EppException.UnimplementedProtocolVersionException} - * @error {@link google.registry.flows.GaeUserCredentials.BadGaeUserIdException} - * @error {@link google.registry.flows.GaeUserCredentials.UserNotLoggedInException} + * @error {@link google.registry.flows.GaeUserCredentials.UserForbiddenException} * @error {@link google.registry.flows.TlsCredentials.BadRegistrarCertificateException} * @error {@link google.registry.flows.TlsCredentials.BadRegistrarIpAddressException} * @error {@link google.registry.flows.TlsCredentials.MissingRegistrarCertificateException} diff --git a/java/google/registry/groups/DirectoryGroupsConnection.java b/java/google/registry/groups/DirectoryGroupsConnection.java index 7c4e0a91a..37e1f6b1f 100644 --- a/java/google/registry/groups/DirectoryGroupsConnection.java +++ b/java/google/registry/groups/DirectoryGroupsConnection.java @@ -46,6 +46,22 @@ public class DirectoryGroupsConnection implements GroupsConnection { private static final String MEMBER_NOT_FOUND_MSG = "Resource Not Found: memberKey"; private static final String MEMBER_ALREADY_EXISTS_MSG = "Member already exists."; + /** + * All possible errors from {@link Directory.Members#get} when an email doesn't belong to a group. + * + *

See {@link #isMemberOfGroup} for details. + * + *

TODO(b/119220829): remove once we transition to using hasMember + * + *

TODO(b/119221854): update error messages if and when they change + */ + private static final ImmutableSet ERROR_MESSAGES_MEMBER_NOT_FOUND = + ImmutableSet.of( + // The given email corresponds to an actual account, but isn't part of this group + "Resource Not Found: memberKey", + // There's no account corresponding to this email + "Missing required field: memberKey"); + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final Groups defaultGroupPermissions = getDefaultGroupPermissions(); @@ -79,7 +95,9 @@ public class DirectoryGroupsConnection implements GroupsConnection { // If the member is already in the group, ignore the error, get the existing member, and // return it. GoogleJsonError err = e.getDetails(); - if (err.getCode() == SC_NOT_FOUND && err.getMessage().equals(GROUP_NOT_FOUND_MSG)) { + if (err == null) { + throw e; + } else if (err.getCode() == SC_NOT_FOUND && err.getMessage().equals(GROUP_NOT_FOUND_MSG)) { logger.atInfo().withCause(e).log( "Creating group %s during addition of member %s because the group doesn't exist.", groupKey, email); @@ -153,7 +171,8 @@ public class DirectoryGroupsConnection implements GroupsConnection { return createdGroup; } catch (GoogleJsonResponseException e) { // Ignore the error thrown if the group already exists. - if (e.getDetails().getCode() == SC_CONFLICT + if (e.getDetails() != null + && e.getDetails().getCode() == SC_CONFLICT && e.getDetails().getMessage().equals("Entity already exists.")) { logger.atInfo().withCause(e).log( "Could not create group %s because it already exists.", groupKey); @@ -163,4 +182,46 @@ public class DirectoryGroupsConnection implements GroupsConnection { } } } + + @Override + public boolean isMemberOfGroup(String memberEmail, String groupKey) { + // We're using "get" instead of "hasMember" because "hasMember" fails for emails that don't + // belong to the G-Suite domain. + // + // "get" fails for users that aren't part of the group, but it also might fail for other + // reasons (no access, group doesn't exist etc.). + // Which error is caused by "user isn't in that group" isn't documented, and was found using + // trial and error. + // + // TODO(b/119221676): transition to using hasMember + // + // Documentation for the API of "get": + // https://developers.google.com/admin-sdk/directory/v1/reference/members/get + // + // Documentation for the API of "hasMember": + // https://developers.google.com/admin-sdk/directory/v1/reference/members/hasMember + try { + Directory.Members.Get getRequest = directory.members().get(groupKey, memberEmail); + Member getReply = getRequest.execute(); + logger.atInfo().log( + "%s is a member of the group %s. Got reply: %s", memberEmail, groupKey, getReply); + return true; + } catch (GoogleJsonResponseException e) { + if (e.getDetails() != null + && ERROR_MESSAGES_MEMBER_NOT_FOUND.contains(e.getDetails().getMessage())) { + // This means the "get" request failed because the email wasn't part of the group. + // This is expected behavior for any visitor that isn't a support group member. + logger.atInfo().log( + "%s isn't a member of the group %s. Got reply %s", + memberEmail, groupKey, e.getMessage()); + return false; + } + // If we got here - we had an unexpected error. Rethrow. + throw new RuntimeException( + String.format("Error checking whether %s is in group %s", memberEmail, groupKey), e); + } catch (IOException e) { + throw new RuntimeException( + String.format("Error checking whether %s is in group %s", memberEmail, groupKey), e); + } + } } diff --git a/java/google/registry/groups/GroupsConnection.java b/java/google/registry/groups/GroupsConnection.java index 23ed0af26..5f7aaf0f9 100644 --- a/java/google/registry/groups/GroupsConnection.java +++ b/java/google/registry/groups/GroupsConnection.java @@ -58,4 +58,7 @@ public interface GroupsConnection { * automatically added as an owner. */ Group createGroup(String groupKey) throws IOException; + + /** Checks whether the given email belongs to the "support" group. */ + boolean isMemberOfGroup(String memberEmail, String groupKey); } diff --git a/java/google/registry/keyring/api/Keyring.java b/java/google/registry/keyring/api/Keyring.java index e600ae529..ff6b51cc4 100644 --- a/java/google/registry/keyring/api/Keyring.java +++ b/java/google/registry/keyring/api/Keyring.java @@ -22,8 +22,8 @@ import org.bouncycastle.openpgp.PGPPublicKey; /** * Nomulus keyring interface. * - *

Separate methods are defined for each specific situation in which the - * registry server needs a secret value, like a PGP key or password. + *

Separate methods are defined for each specific situation in which the registry server needs a + * secret value, like a PGP key or password. */ @ThreadSafe public interface Keyring extends AutoCloseable { @@ -31,11 +31,10 @@ public interface Keyring extends AutoCloseable { /** * Returns the key which should be used to sign RDE deposits being uploaded to a third-party. * - *

When we give all our data to the escrow provider, they'll need - * a signature to ensure the data is authentic. + *

When we give all our data to the escrow provider, they'll need a signature to ensure the + * data is authentic. * - *

This keypair should only be known to the domain registry shared - * registry system. + *

This keypair should only be known to the domain registry shared registry system. * * @see google.registry.rde.RdeUploadAction */ @@ -44,12 +43,10 @@ public interface Keyring extends AutoCloseable { /** * Returns public key for encrypting escrow deposits being staged to cloud storage. * - *

This adds an additional layer of security so cloud storage administrators - * won't be tempted to go poking around the App Engine Cloud Console and see a - * dump of the entire database. + *

This adds an additional layer of security so cloud storage administrators won't be tempted + * to go poking around the App Engine Cloud Console and see a dump of the entire database. * - *

This keypair should only be known to the domain registry shared - * registry system. + *

This keypair should only be known to the domain registry shared registry system. * * @see #getRdeStagingDecryptionKey() */ @@ -58,10 +55,9 @@ public interface Keyring extends AutoCloseable { /** * Returns private key for decrypting escrow deposits retrieved from cloud storage. * - *

This method may impose restrictions on who can call it. For example, we'd want - * to check that the caller isn't an HTTP request attacking a vulnerability in the - * admin console. The request should originate from a backend task queue servlet - * invocation of the RDE upload thing. + *

This method may impose restrictions on who can call it. For example, we'd want to check that + * the caller isn't an HTTP request attacking a vulnerability in the admin console. The request + * should originate from a backend task queue servlet invocation of the RDE upload thing. * * @see #getRdeStagingEncryptionKey() * @see google.registry.rde.RdeUploadAction @@ -92,9 +88,9 @@ public interface Keyring extends AutoCloseable { /** * Returns public key for SSH client connections made by RDE. * - *

This is a string containing what would otherwise be the contents of an - * {@code ~/.ssh/id_rsa.pub} file. It's usually a single line with the name of - * the algorithm, the base64 key, and the email address of the owner. + *

This is a string containing what would otherwise be the contents of an {@code + * ~/.ssh/id_rsa.pub} file. It's usually a single line with the name of the algorithm, the base64 + * key, and the email address of the owner. * * @see google.registry.rde.RdeUploadAction */ @@ -103,13 +99,12 @@ public interface Keyring extends AutoCloseable { /** * Returns private key for SSH client connections made by RDE. * - *

This is a string containing what would otherwise be the contents of an - * {@code ~/.ssh/id_rsa} file. It's ASCII-armored text. + *

This is a string containing what would otherwise be the contents of an {@code ~/.ssh/id_rsa} + * file. It's ASCII-armored text. * - *

This method may impose restrictions on who can call it. For example, we'd want - * to check that the caller isn't an HTTP request attacking a vulnerability in the - * admin console. The request should originate from a backend task queue servlet - * invocation of the RDE upload thing. + *

This method may impose restrictions on who can call it. For example, we'd want to check that + * the caller isn't an HTTP request attacking a vulnerability in the admin console. The request + * should originate from a backend task queue servlet invocation of the RDE upload thing. * * @see google.registry.rde.RdeUploadAction */ diff --git a/java/google/registry/keyring/kms/KmsKeyring.java b/java/google/registry/keyring/kms/KmsKeyring.java index 262145f91..e8968784f 100644 --- a/java/google/registry/keyring/kms/KmsKeyring.java +++ b/java/google/registry/keyring/kms/KmsKeyring.java @@ -41,6 +41,7 @@ import org.bouncycastle.openpgp.PGPPublicKey; */ public class KmsKeyring implements Keyring { + /** Key labels for private key secrets. */ enum PrivateKeyLabel { BRDA_SIGNING_PRIVATE, RDE_SIGNING_PRIVATE, @@ -51,6 +52,7 @@ public class KmsKeyring implements Keyring { } } + /** Key labels for public key secrets. */ enum PublicKeyLabel { BRDA_RECEIVER_PUBLIC, BRDA_SIGNING_PUBLIC, @@ -63,6 +65,7 @@ public class KmsKeyring implements Keyring { } } + /** Key labels for string secrets. */ enum StringKeyLabel { SAFE_BROWSING_API_KEY, ICANN_REPORTING_PASSWORD_STRING, diff --git a/java/google/registry/model/EppResourceUtils.java b/java/google/registry/model/EppResourceUtils.java index dba9b9b4c..3de360c28 100644 --- a/java/google/registry/model/EppResourceUtils.java +++ b/java/google/registry/model/EppResourceUtils.java @@ -44,6 +44,7 @@ import google.registry.model.transfer.TransferData; import google.registry.model.transfer.TransferStatus; import java.util.List; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import javax.annotation.Nullable; @@ -75,7 +76,7 @@ public final class EppResourceUtils { /** * Loads the last created version of an {@link EppResource} from Datastore by foreign key. * - *

Returns null if no resource with this foreign key was ever created, or if the most recently + *

Returns empty if no resource with this foreign key was ever created, or if the most recently * created resource was deleted before time "now". * *

Loading an {@link EppResource} by itself is not sufficient to know its current state since @@ -91,8 +92,7 @@ public final class EppResourceUtils { * @param foreignKey id to match * @param now the current logical time to project resources at */ - @Nullable - public static T loadByForeignKey( + public static Optional loadByForeignKey( Class clazz, String foreignKey, DateTime now) { return loadByForeignKeyHelper(clazz, foreignKey, now, false); } @@ -120,15 +120,13 @@ public final class EppResourceUtils { * @param foreignKey id to match * @param now the current logical time to project resources at */ - @Nullable - public static T loadByForeignKeyCached( + public static Optional loadByForeignKeyCached( Class clazz, String foreignKey, DateTime now) { return loadByForeignKeyHelper( clazz, foreignKey, now, RegistryConfig.isEppResourceCachingEnabled()); } - @Nullable - private static T loadByForeignKeyHelper( + private static Optional loadByForeignKeyHelper( Class clazz, String foreignKey, DateTime now, boolean useCache) { checkArgument( ForeignKeyedEppResource.class.isAssignableFrom(clazz), @@ -140,14 +138,14 @@ public final class EppResourceUtils { : ofy().load().type(ForeignKeyIndex.mapToFkiClass(clazz)).id(foreignKey).now(); // The value of fki.getResourceKey() might be null for hard-deleted prober data. if (fki == null || isAtOrAfter(now, fki.getDeletionTime()) || fki.getResourceKey() == null) { - return null; + return Optional.empty(); } T resource = useCache ? EppResource.loadCached(fki.getResourceKey()) : ofy().load().key(fki.getResourceKey()).now(); if (resource == null || isAtOrAfter(now, resource.getDeletionTime())) { - return null; + return Optional.empty(); } // When setting status values based on a time, choose the greater of "now" and the resource's // UpdateAutoTimestamp. For non-mutating uses (info, whois, etc.), this is equivalent to rolling @@ -155,24 +153,25 @@ public final class EppResourceUtils { // doesn't appear stale. For mutating flows, if we had to roll now forward then the flow will // fail when it tries to save anything via Ofy, since "now" is needed to be > the last update // time for writes. - return cloneProjectedAtTime( - resource, latestOf(now, resource.getUpdateAutoTimestamp().getTimestamp())); + return Optional.of( + cloneProjectedAtTime( + resource, latestOf(now, resource.getUpdateAutoTimestamp().getTimestamp()))); } /** - * Returns the domain application with the given application id if it exists, or null if it does + * Returns the domain application with the given application id if it exists, or absent if it does * not or is soft-deleted as of the given time. */ - @Nullable - public static DomainApplication loadDomainApplication(String applicationId, DateTime now) { + public static Optional loadDomainApplication( + String applicationId, DateTime now) { DomainApplication application = ofy().load().key(Key.create(DomainApplication.class, applicationId)).now(); if (application == null || isAtOrAfter(now, application.getDeletionTime())) { - return null; + return Optional.empty(); } // Applications don't have any speculative changes that become effective later, so no need to // clone forward in time. - return application; + return Optional.of(application); } /** @@ -402,13 +401,5 @@ public final class EppResourceUtils { return queryForLinkedDomains(key, now).limit(1).count() > 0; } - /** Exception to throw when failing to parse a repo id. */ - public static class InvalidRepoIdException extends Exception { - - public InvalidRepoIdException(String message) { - super(message); - } - } - private EppResourceUtils() {} } diff --git a/java/google/registry/model/OteAccountBuilder.java b/java/google/registry/model/OteAccountBuilder.java new file mode 100644 index 000000000..52997c497 --- /dev/null +++ b/java/google/registry/model/OteAccountBuilder.java @@ -0,0 +1,390 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.DateTimeUtils.START_OF_TIME; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.Sets; +import com.google.common.collect.Streams; +import com.googlecode.objectify.Key; +import google.registry.config.RegistryEnvironment; +import google.registry.model.common.GaeUserIdConverter; +import google.registry.model.pricing.StaticPremiumListPricingEngine; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.RegistrarAddress; +import google.registry.model.registrar.RegistrarContact; +import google.registry.model.registry.Registry; +import google.registry.model.registry.Registry.TldState; +import google.registry.model.registry.label.PremiumList; +import google.registry.util.CidrAddressBlock; +import java.util.Collection; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Pattern; +import org.joda.money.CurrencyUnit; +import org.joda.money.Money; +import org.joda.time.DateTime; +import org.joda.time.Duration; + +/** + * Class to help build and persist all the OT&E entities in Datastore. + * + *

This includes the TLDs (Registries), Registrars, and the RegistrarContacts that can access the + * web console. + * + * This class is basically a "builder" for the parameters needed to generate the OT&E entities. + * Nothing is created until you call {@link #buildAndPersist}. + * + * Usage example: + * + * 

   {@code
+ * OteAccountBuilder.forClientId("example")
+ *     .addContact("contact@email.com") // OPTIONAL
+ *     .setPassword("password") // OPTIONAL
+ *     .setCertificateHash(certificateHash) // OPTIONAL
+ *     .setIpWhitelist(ImmutableList.of("1.1.1.1", "2.2.2.0/24")) // OPTIONAL
+ *     .buildAndPersist();
+ * }
+ */ +public final class OteAccountBuilder { + + // Regex: 3-14 lower-case alphanumeric characters or hyphens, the first of which must be a letter. + private static final Pattern REGISTRAR_PATTERN = Pattern.compile("^[a-z][-a-z0-9]{2,13}$"); + + // Durations are short so that registrars can test with quick transfer (etc.) turnaround. + private static final Duration SHORT_ADD_GRACE_PERIOD = Duration.standardMinutes(60); + private static final Duration SHORT_REDEMPTION_GRACE_PERIOD = Duration.standardMinutes(10); + private static final Duration SHORT_PENDING_DELETE_LENGTH = Duration.standardMinutes(5); + + private static final String DEFAULT_PREMIUM_LIST = "default_sandbox_list"; + + private static final RegistrarAddress DEFAULT_ADDRESS = + new RegistrarAddress.Builder() + .setStreet(ImmutableList.of("e-street")) + .setCity("Neverland") + .setState("NY") + .setCountryCode("US") + .setZip("55555") + .build(); + + private static final ImmutableSortedMap EAP_FEE_SCHEDULE = + ImmutableSortedMap.of( + new DateTime(0), + Money.of(CurrencyUnit.USD, 0), + DateTime.parse("2018-03-01T00:00:00Z"), + Money.of(CurrencyUnit.USD, 100), + DateTime.parse("2030-03-01T00:00:00Z"), + Money.of(CurrencyUnit.USD, 0)); + + private final ImmutableMap clientIdToTld; + private final Registry sunriseTld; + private final Registry landrushTld; + private final Registry gaTld; + private final Registry eapTld; + private final ImmutableList.Builder contactsBuilder = + new ImmutableList.Builder<>(); + + private ImmutableList registrars; + private boolean replaceExisting = false; + + private OteAccountBuilder(String baseClientId) { + checkState( + RegistryEnvironment.get() != RegistryEnvironment.PRODUCTION, + "Can't setup OT&E in production"); + clientIdToTld = createClientIdToTldMap(baseClientId); + sunriseTld = + createTld( + baseClientId + "-sunrise", TldState.START_DATE_SUNRISE, null, null, null, false, 0); + landrushTld = + createTld(baseClientId + "-landrush", TldState.LANDRUSH, null, null, null, false, 1); + gaTld = + createTld( + baseClientId + "-ga", + TldState.GENERAL_AVAILABILITY, + SHORT_ADD_GRACE_PERIOD, + SHORT_REDEMPTION_GRACE_PERIOD, + SHORT_PENDING_DELETE_LENGTH, + false, + 2); + eapTld = + createTld( + baseClientId + "-eap", + TldState.GENERAL_AVAILABILITY, + SHORT_ADD_GRACE_PERIOD, + SHORT_REDEMPTION_GRACE_PERIOD, + SHORT_PENDING_DELETE_LENGTH, + true, + 3); + registrars = + clientIdToTld.keySet().stream() + .map(OteAccountBuilder::createRegistrar) + .collect(toImmutableList()); + } + + /** + * Creates an OteAccountBuilder for the given base client ID. + * + * @param baseClientId the base clientId which will help name all the entities we create. Normally + * is the same as the "prod" clientId designated for this registrar. + */ + public static OteAccountBuilder forClientId(String baseClientId) { + return new OteAccountBuilder(baseClientId); + } + + /** + * Set whether to replace any conflicting existing entities. + * + *

If true, any existing entity that conflicts with the entities we want to create will be + * replaced with the newly created data. + * + *

If false, encountering an existing entity that conflicts with one we want to create will + * throw an exception during {@link #buildAndPersist}. + * + *

NOTE that if we fail, no entities are created (the creation is atomic). + * + *

Default is false (failing if entities exist) + */ + public OteAccountBuilder setReplaceExisting(boolean replaceExisting) { + this.replaceExisting = replaceExisting; + return this; + } + + /** + * Adds a RegistrarContact with Web Console access. + * + *

NOTE: can be called more than once, adding multiple contacts. Each contact will have access + * to all OT&E Registrars. + * + * @param email the contact email that will have web-console access to all the Registrars. Must be + * from "our G Suite domain" (we have to be able to get its GaeUserId) + */ + public OteAccountBuilder addContact(String email) { + String gaeUserId = + checkNotNull( + GaeUserIdConverter.convertEmailAddressToGaeUserId(email), + "Email address %s is not associated with any GAE ID", + email); + registrars.forEach( + registrar -> contactsBuilder.add(createRegistrarContact(email, gaeUserId, registrar))); + return this; + } + + /** + * Apply a function on all the OT&E Registrars. + * + *

Use this to set up registrar fields. + * + *

NOTE: DO NOT change anything that would affect the {@link Key#create} result on Registrars. + * If you want to make this function public, add a check that the Key.create on the registrars + * hasn't changed. + * + * @param func a function setting the requested fields on Registrar Builders. Will be applied to + * all the Registrars. + */ + private OteAccountBuilder transformRegistrars( + Function func) { + registrars = + registrars.stream() + .map(Registrar::asBuilder) + .map(func) + .map(Registrar.Builder::build) + .collect(toImmutableList()); + return this; + } + + /** Sets the EPP login password for all the OT&E Registrars. */ + public OteAccountBuilder setPassword(String password) { + return transformRegistrars(builder -> builder.setPassword(password)); + } + + /** Sets the client certificate hash to all the OT&E Registrars. */ + public OteAccountBuilder setCertificateHash(String certHash) { + return transformRegistrars(builder -> builder.setClientCertificateHash(certHash)); + } + + /** Sets the client certificate to all the OT&E Registrars. */ + public OteAccountBuilder setCertificate(String asciiCert, DateTime now) { + return transformRegistrars(builder -> builder.setClientCertificate(asciiCert, now)); + } + + /** Sets the IP whitelist to all the OT&E Registrars. */ + public OteAccountBuilder setIpWhitelist(Collection ipWhitelist) { + ImmutableList ipAddressWhitelist = + ipWhitelist.stream().map(CidrAddressBlock::create).collect(toImmutableList()); + return transformRegistrars(builder -> builder.setIpAddressWhitelist(ipAddressWhitelist)); + } + + /** + * Persists all the OT&E entities to datastore. + * + * @return map from the new clientIds created to the new TLDs they have access to. Can be used to + * go over all the newly created Registrars / Registries / RegistrarContacts if any + * post-creation work is needed. + */ + public ImmutableMap buildAndPersist() { + // save all the entitiesl in a single transaction + ofy().transact(this::saveAllEntities); + return clientIdToTld; + } + + /** + * Return map from the OT&E clientIds we will create to the new TLDs they will have access to. + */ + public ImmutableMap getClientIdToTldMap() { + return clientIdToTld; + } + + /** Saves all the OT&E entities we created. */ + private void saveAllEntities() { + ofy().assertInTransaction(); + + ImmutableList registries = ImmutableList.of(sunriseTld, landrushTld, gaTld, eapTld); + ImmutableList contacts = contactsBuilder.build(); + + if (!replaceExisting) { + ImmutableList> keys = + Streams.concat(registries.stream(), registrars.stream(), contacts.stream()) + .map(Key::create) + .collect(toImmutableList()); + Set> existingKeys = ofy().load().keys(keys).keySet(); + checkState( + existingKeys.isEmpty(), + "Found existing object(s) conflicting with OT&E objects: %s", + existingKeys); + } + // Save the Registries (TLDs) first + ofy().save().entities(registries).now(); + // Now we can set the allowedTlds for the registrars + registrars = registrars.stream().map(this::addAllowedTld).collect(toImmutableList()); + // and we can save the registrars and contacts! + ofy().save().entities(registrars); + ofy().save().entities(contacts); + } + + private Registrar addAllowedTld(Registrar registrar) { + String tld = clientIdToTld.get(registrar.getClientId()); + if (registrar.getAllowedTlds().contains(tld)) { + return registrar; + } + return registrar + .asBuilder() + .setAllowedTldsUncached(Sets.union(registrar.getAllowedTlds(), ImmutableSet.of(tld))) + .build(); + } + + private static Registry createTld( + String tldName, + TldState initialTldState, + Duration addGracePeriod, + Duration redemptionGracePeriod, + Duration pendingDeleteLength, + boolean isEarlyAccess, + int roidSuffix) { + String tldNameAlphaNumerical = tldName.replaceAll("[^a-z0-9]", ""); + Optional premiumList = PremiumList.getUncached(DEFAULT_PREMIUM_LIST); + checkState(premiumList.isPresent(), "Couldn't find premium list %s.", DEFAULT_PREMIUM_LIST); + Registry.Builder builder = + new Registry.Builder() + .setTldStr(tldName) + .setPremiumPricingEngine(StaticPremiumListPricingEngine.NAME) + .setTldStateTransitions(ImmutableSortedMap.of(START_OF_TIME, initialTldState)) + .setDnsWriters(ImmutableSet.of("VoidDnsWriter")) + .setPremiumList(premiumList.get()) + .setRoidSuffix( + String.format( + "%S%X", + tldNameAlphaNumerical.substring(0, Math.min(tldNameAlphaNumerical.length(), 7)), + roidSuffix)); + if (addGracePeriod != null) { + builder.setAddGracePeriodLength(addGracePeriod); + } + if (pendingDeleteLength != null) { + builder.setPendingDeleteLength(pendingDeleteLength); + } + if (redemptionGracePeriod != null) { + builder.setRedemptionGracePeriodLength(redemptionGracePeriod); + } + if (isEarlyAccess) { + builder.setEapFeeSchedule(EAP_FEE_SCHEDULE); + } + return builder.build(); + } + + /** + * Creates the Registrar without the allowedTlds set - because we can't set allowedTlds before the + * TLD is saved. + */ + private static Registrar createRegistrar(String registrarName) { + return new Registrar.Builder() + .setClientId(registrarName) + .setRegistrarName(registrarName) + .setType(Registrar.Type.OTE) + .setLocalizedAddress(DEFAULT_ADDRESS) + .setEmailAddress("foo@neverland.com") + .setFaxNumber("+1.2125550100") + .setPhoneNumber("+1.2125550100") + .setIcannReferralEmail("nightmare@registrar.test") + .setState(Registrar.State.ACTIVE) + .build(); + } + + private static RegistrarContact createRegistrarContact( + String email, String gaeUserId, Registrar registrar) { + return new RegistrarContact.Builder() + .setParent(registrar) + .setName(email) + .setEmailAddress(email) + .setGaeUserId(gaeUserId) + .build(); + } + + /** Returns the ClientIds of the OT&E, with the TLDs each has access to. */ + public static ImmutableMap createClientIdToTldMap(String baseClientId) { + checkArgument( + REGISTRAR_PATTERN.matcher(baseClientId).matches(), + "Invalid registrar name: %s", + baseClientId); + return new ImmutableMap.Builder() + .put(baseClientId + "-1", baseClientId + "-sunrise") + .put(baseClientId + "-2", baseClientId + "-landrush") + .put(baseClientId + "-3", baseClientId + "-ga") + .put(baseClientId + "-4", baseClientId + "-ga") + .put(baseClientId + "-5", baseClientId + "-eap") + .build(); + } + + /** Returns the base client ID that correspond to a given OT&E client ID. */ + public static String getBaseClientId(String oteClientId) { + int index = oteClientId.lastIndexOf('-'); + checkArgument(index > 0, "Invalid OT&E client ID: %s", oteClientId); + String baseClientId = oteClientId.substring(0, index); + checkArgument( + createClientIdToTldMap(baseClientId).containsKey(oteClientId), + "ID %s is not one of the OT&E client IDs for base %s", + oteClientId, + baseClientId); + return baseClientId; + } +} diff --git a/java/google/registry/model/OteStats.java b/java/google/registry/model/OteStats.java new file mode 100644 index 000000000..55301624b --- /dev/null +++ b/java/google/registry/model/OteStats.java @@ -0,0 +1,277 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model; + +import static com.google.common.base.Predicates.equalTo; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.model.eppcommon.EppXmlTransformer.unmarshal; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.CollectionUtils.isNullOrEmpty; +import static google.registry.util.DomainNameUtils.ACE_PREFIX; + +import com.google.common.base.Ascii; +import com.google.common.base.Predicates; +import com.google.common.collect.HashMultiset; +import com.google.common.collect.ImmutableCollection; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multiset; +import com.googlecode.objectify.Key; +import com.googlecode.objectify.cmd.Query; +import google.registry.model.domain.DomainCommand; +import google.registry.model.domain.fee.FeeCreateCommandExtension; +import google.registry.model.domain.launch.LaunchCreateExtension; +import google.registry.model.domain.secdns.SecDnsCreateExtension; +import google.registry.model.domain.secdns.SecDnsUpdateExtension; +import google.registry.model.eppinput.EppInput; +import google.registry.model.eppinput.EppInput.ResourceCommandWrapper; +import google.registry.model.host.HostCommand; +import google.registry.model.reporting.HistoryEntry; +import google.registry.model.reporting.HistoryEntry.Type; +import google.registry.xml.XmlException; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** Represents stats derived from HistoryEntry objects on actions taken by registrars. */ +public class OteStats { + + /** + * Returns the statistics about the OT&E actions that have been taken by a particular registrar. + */ + public static OteStats getFromRegistrar(String registrarName) { + return new OteStats().recordRegistrarHistory(registrarName); + } + + private OteStats() {} + + private static final Predicate HAS_CLAIMS_NOTICE = + eppInput -> { + Optional launchCreate = + eppInput.getSingleExtension(LaunchCreateExtension.class); + return launchCreate.isPresent() && launchCreate.get().getNotice() != null; + }; + + private static final Predicate HAS_SEC_DNS = + eppInput -> + eppInput.getSingleExtension(SecDnsCreateExtension.class).isPresent() + || eppInput.getSingleExtension(SecDnsUpdateExtension.class).isPresent(); + + private static final Predicate IS_SUNRISE = + eppInput -> { + Optional launchCreate = + eppInput.getSingleExtension(LaunchCreateExtension.class); + return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks()); + }; + + private static final Predicate IS_IDN = + eppInput -> + ((DomainCommand.Create) + ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) + .getResourceCommand()) + .getFullyQualifiedDomainName() + .startsWith(ACE_PREFIX); + + private static final Predicate IS_SUBORDINATE = + eppInput -> + !isNullOrEmpty( + ((HostCommand.Create) + ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) + .getResourceCommand()) + .getInetAddresses()); + + /** Enum defining the distinct statistics (types of registrar actions) to record. */ + public enum StatType { + CONTACT_CREATES(0, equalTo(Type.CONTACT_CREATE)), + CONTACT_DELETES(0, equalTo(Type.CONTACT_DELETE)), + CONTACT_TRANSFER_APPROVES(0, equalTo(Type.CONTACT_TRANSFER_APPROVE)), + CONTACT_TRANSFER_CANCELS(0, equalTo(Type.CONTACT_TRANSFER_CANCEL)), + CONTACT_TRANSFER_REJECTS(0, equalTo(Type.CONTACT_TRANSFER_REJECT)), + CONTACT_TRANSFER_REQUESTS(0, equalTo(Type.CONTACT_TRANSFER_REQUEST)), + CONTACT_UPDATES(0, equalTo(Type.CONTACT_UPDATE)), + DOMAIN_APPLICATION_CREATES(0, equalTo(Type.DOMAIN_APPLICATION_CREATE)), + DOMAIN_APPLICATION_CREATES_LANDRUSH( + 0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE.negate()), + DOMAIN_APPLICATION_CREATES_SUNRISE(0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE), + DOMAIN_APPLICATION_DELETES(0, equalTo(Type.DOMAIN_APPLICATION_DELETE)), + DOMAIN_APPLICATION_UPDATES(0, equalTo(Type.DOMAIN_APPLICATION_UPDATE)), + DOMAIN_AUTORENEWS(0, equalTo(Type.DOMAIN_AUTORENEW)), + DOMAIN_CREATES(0, equalTo(Type.DOMAIN_CREATE)), + DOMAIN_CREATES_ASCII(1, equalTo(Type.DOMAIN_CREATE), IS_IDN.negate()), + DOMAIN_CREATES_IDN(1, equalTo(Type.DOMAIN_CREATE), IS_IDN), + DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), IS_SUNRISE), + DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), HAS_CLAIMS_NOTICE), + DOMAIN_CREATES_WITH_FEE( + 1, + equalTo(Type.DOMAIN_CREATE), + eppInput -> eppInput.getSingleExtension(FeeCreateCommandExtension.class).isPresent()), + DOMAIN_CREATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS), + DOMAIN_CREATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS.negate()), + DOMAIN_DELETES(2, equalTo(Type.DOMAIN_DELETE)), + DOMAIN_RENEWS(0, equalTo(Type.DOMAIN_RENEW)), + DOMAIN_RESTORES(1, equalTo(Type.DOMAIN_RESTORE)), + DOMAIN_TRANSFER_APPROVES(1, equalTo(Type.DOMAIN_TRANSFER_APPROVE)), + DOMAIN_TRANSFER_CANCELS(1, equalTo(Type.DOMAIN_TRANSFER_CANCEL)), + DOMAIN_TRANSFER_REJECTS(1, equalTo(Type.DOMAIN_TRANSFER_REJECT)), + DOMAIN_TRANSFER_REQUESTS(1, equalTo(Type.DOMAIN_TRANSFER_REQUEST)), + DOMAIN_UPDATES(0, equalTo(Type.DOMAIN_UPDATE)), + DOMAIN_UPDATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS), + DOMAIN_UPDATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS.negate()), + HOST_CREATES(0, equalTo(Type.HOST_CREATE)), + HOST_CREATES_EXTERNAL(0, equalTo(Type.HOST_CREATE), IS_SUBORDINATE.negate()), + HOST_CREATES_SUBORDINATE(1, equalTo(Type.HOST_CREATE), IS_SUBORDINATE), + HOST_DELETES(1, equalTo(Type.HOST_DELETE)), + HOST_UPDATES(1, equalTo(Type.HOST_UPDATE)), + UNCLASSIFIED_FLOWS(0, Predicates.alwaysFalse()); + + /** StatTypes with a non-zero requirement */ + public static final ImmutableList REQUIRED_STAT_TYPES = + Arrays.stream(values()) + .filter(statType -> statType.requirement > 0) + .collect(toImmutableList()); + + /** Required number of times registrars must complete this action. */ + private final int requirement; + + /** Filter to check the HistoryEntry Type */ + @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. + private final Predicate typeFilter; + + /** Optional filter on the EppInput. */ + @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. + private final Optional> eppInputFilter; + + StatType(int requirement, Predicate typeFilter) { + this(requirement, typeFilter, null); + } + + StatType( + int requirement, + Predicate typeFilter, + Predicate eppInputFilter) { + this.requirement = requirement; + this.typeFilter = typeFilter; + if (eppInputFilter == null) { + this.eppInputFilter = Optional.empty(); + } else { + this.eppInputFilter = Optional.of(eppInputFilter); + } + } + + /** Returns the number of times this StatType must be performed. */ + public int getRequirement() { + return requirement; + } + + /** Returns a more human-readable translation of the enum constant. */ + public String getDescription() { + return Ascii.toLowerCase(this.name().replace('_', ' ')); + } + + /** + * Check if the {@link HistoryEntry} type matches as well as the {@link EppInput} if supplied. + */ + private boolean matches(HistoryEntry.Type historyType, Optional eppInput) { + if (eppInputFilter.isPresent() && eppInput.isPresent()) { + return typeFilter.test(historyType) && eppInputFilter.get().test(eppInput.get()); + } else { + return typeFilter.test(historyType); + } + } + } + + /** Stores counts of how many times each action type was performed. */ + private final Multiset statCounts = HashMultiset.create(); + + /** + * Records data on what actions have been performed by the four numbered OT&E variants of the + * registrar name. + * + *

Stops when it notices that all tests have passed. + */ + private OteStats recordRegistrarHistory(String registrarName) { + ImmutableCollection clientIds = + OteAccountBuilder.createClientIdToTldMap(registrarName).keySet(); + + Query query = + ofy() + .load() + .type(HistoryEntry.class) + .filter("clientId in", clientIds) + .order("modificationTime"); + for (HistoryEntry historyEntry : query) { + try { + record(historyEntry); + } catch (XmlException e) { + throw new RuntimeException("Couldn't parse history entry " + Key.create(historyEntry), e); + } + // Break out early if all tests were passed. + if (wereAllTestsPassed()) { + break; + } + } + return this; + } + + /** Interprets the data in the provided HistoryEntry and increments counters. */ + private void record(final HistoryEntry historyEntry) throws XmlException { + byte[] xmlBytes = historyEntry.getXmlBytes(); + // xmlBytes can be null on contact create and update for safe-harbor compliance. + final Optional eppInput = + (xmlBytes == null) ? Optional.empty() : Optional.of(unmarshal(EppInput.class, xmlBytes)); + if (!statCounts.addAll( + EnumSet.allOf(StatType.class).stream() + .filter(statType -> statType.matches(historyEntry.getType(), eppInput)) + .collect(toImmutableList()))) { + statCounts.add(StatType.UNCLASSIFIED_FLOWS); + } + } + + private boolean wereAllTestsPassed() { + return Arrays.stream(StatType.values()).allMatch(s -> statCounts.count(s) >= s.requirement); + } + + /** Returns the total number of actions taken */ + public int getSize() { + return statCounts.size(); + } + + /** Returns the number of times that a particular StatType was seen */ + public int getCount(StatType statType) { + return statCounts.count(statType); + } + + /** + * Returns a list of failures, any cases where the passed stats fail to meet the required + * thresholds, or the empty list if all requirements are met. + */ + public ImmutableList getFailures() { + return StatType.REQUIRED_STAT_TYPES.stream() + .filter(statType -> statCounts.count(statType) < statType.requirement) + .collect(toImmutableList()); + } + + /** Returns a string showing all possible actions and how many times each was performed. */ + @Override + public String toString() { + return String.format( + "%s\nTOTAL: %d", + EnumSet.allOf(StatType.class).stream() + .map(stat -> String.format("%s: %d", stat.getDescription(), statCounts.count(stat))) + .collect(Collectors.joining("\n")), + statCounts.size()); + } +} diff --git a/java/google/registry/model/common/GaeUserIdConverter.java b/java/google/registry/model/common/GaeUserIdConverter.java index a4fc98305..57d08e156 100644 --- a/java/google/registry/model/common/GaeUserIdConverter.java +++ b/java/google/registry/model/common/GaeUserIdConverter.java @@ -14,6 +14,7 @@ package google.registry.model.common; +import static com.google.common.base.Preconditions.checkState; import static google.registry.model.ofy.ObjectifyService.allocateId; import static google.registry.model.ofy.ObjectifyService.ofy; @@ -24,6 +25,7 @@ import com.googlecode.objectify.annotation.Id; import google.registry.model.ImmutableObject; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; +import java.util.List; /** * A helper class to convert email addresses to GAE user ids. It does so by persisting a User @@ -46,8 +48,9 @@ public class GaeUserIdConverter extends ImmutableObject { public static String convertEmailAddressToGaeUserId(String emailAddress) { final GaeUserIdConverter gaeUserIdConverter = new GaeUserIdConverter(); gaeUserIdConverter.id = allocateId(); - gaeUserIdConverter.user = - new User(emailAddress, Splitter.on('@').splitToList(emailAddress).get(1)); + List emailParts = Splitter.on('@').splitToList(emailAddress); + checkState(emailParts.size() == 2, "'%s' is not a valid email address", emailAddress); + gaeUserIdConverter.user = new User(emailAddress, emailParts.get(1)); try { // Perform these operations in a transactionless context to avoid enlisting in some outer diff --git a/java/google/registry/model/domain/DomainBase.java b/java/google/registry/model/domain/DomainBase.java index 022a42cd5..110cc2a92 100644 --- a/java/google/registry/model/domain/DomainBase.java +++ b/java/google/registry/model/domain/DomainBase.java @@ -21,6 +21,7 @@ import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; import static com.google.common.collect.Sets.difference; import static com.google.common.collect.Sets.union; import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.CollectionUtils.forceEmptyToNull; import static google.registry.util.CollectionUtils.nullToEmpty; import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy; import static google.registry.util.CollectionUtils.nullToEmptyImmutableSortedCopy; @@ -231,21 +232,38 @@ public abstract class DomainBase extends EppResource { return thisCastToDerived(); } - public B setNameservers(ImmutableSet> nameservers) { - getInstance().nsHosts = nameservers; + public B setNameservers(Key nameserver) { + getInstance().nsHosts = ImmutableSet.of(nameserver); return thisCastToDerived(); } + public B setNameservers(ImmutableSet> nameservers) { + getInstance().nsHosts = forceEmptyToNull(nameservers); + return thisCastToDerived(); + } + + public B addNameserver(Key nameserver) { + return addNameservers(ImmutableSet.of(nameserver)); + } + public B addNameservers(ImmutableSet> nameservers) { return setNameservers( ImmutableSet.copyOf(union(getInstance().getNameservers(), nameservers))); } + public B removeNameserver(Key nameserver) { + return removeNameservers(ImmutableSet.of(nameserver)); + } + public B removeNameservers(ImmutableSet> nameservers) { return setNameservers( ImmutableSet.copyOf(difference(getInstance().getNameservers(), nameservers))); } + public B setContacts(DesignatedContact contact) { + return setContacts(ImmutableSet.of(contact)); + } + public B setContacts(ImmutableSet contacts) { checkArgument(contacts.stream().noneMatch(IS_REGISTRANT), "Registrant cannot be a contact"); // Replace the non-registrant contacts inside allContacts. diff --git a/java/google/registry/model/domain/DomainResource.java b/java/google/registry/model/domain/DomainResource.java index aee074696..510d75a76 100644 --- a/java/google/registry/model/domain/DomainResource.java +++ b/java/google/registry/model/domain/DomainResource.java @@ -281,10 +281,15 @@ public class DomainResource extends DomainBase } // Set all remaining transfer properties. setAutomaticTransferSuccessProperties(builder, transferData); + builder + .setLastEppUpdateTime(transferExpirationTime) + .setLastEppUpdateClientId(transferData.getGainingClientId()); // Finish projecting to now. return builder.build().cloneProjectedAtTime(now); } + Optional newLastEppUpdateTime = Optional.empty(); + // There is no transfer. Do any necessary autorenews. Builder builder = asBuilder(); @@ -296,11 +301,13 @@ public class DomainResource extends DomainBase DateTime newExpirationTime = lastAutorenewTime.plusYears(1); builder .setRegistrationExpirationTime(newExpirationTime) - .addGracePeriod(GracePeriod.createForRecurring( - GracePeriodStatus.AUTO_RENEW, - lastAutorenewTime.plus(Registry.get(getTld()).getAutoRenewGracePeriodLength()), - getCurrentSponsorClientId(), - autorenewBillingEvent)); + .addGracePeriod( + GracePeriod.createForRecurring( + GracePeriodStatus.AUTO_RENEW, + lastAutorenewTime.plus(Registry.get(getTld()).getAutoRenewGracePeriodLength()), + getCurrentSponsorClientId(), + autorenewBillingEvent)); + newLastEppUpdateTime = Optional.of(lastAutorenewTime); } // Remove any grace periods that have expired. @@ -309,6 +316,22 @@ public class DomainResource extends DomainBase for (GracePeriod gracePeriod : almostBuilt.getGracePeriods()) { if (isBeforeOrAt(gracePeriod.getExpirationTime(), now)) { builder.removeGracePeriod(gracePeriod); + if (!newLastEppUpdateTime.isPresent() + || isBeforeOrAt(newLastEppUpdateTime.get(), gracePeriod.getExpirationTime())) { + newLastEppUpdateTime = Optional.of(gracePeriod.getExpirationTime()); + } + } + } + + // It is possible that the lastEppUpdateClientId is different from current sponsor client + // id, so we have to do the comparison instead of having one variable just storing the most + // recent time. + if (newLastEppUpdateTime.isPresent()) { + if (getLastEppUpdateTime() == null + || newLastEppUpdateTime.get().isAfter(getLastEppUpdateTime())) { + builder + .setLastEppUpdateTime(newLastEppUpdateTime.get()) + .setLastEppUpdateClientId(getCurrentSponsorClientId()); } } diff --git a/java/google/registry/model/eppcommon/EppXmlTransformer.java b/java/google/registry/model/eppcommon/EppXmlTransformer.java new file mode 100644 index 000000000..b4a3a0c2a --- /dev/null +++ b/java/google/registry/model/eppcommon/EppXmlTransformer.java @@ -0,0 +1,95 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model.eppcommon; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import google.registry.model.ImmutableObject; +import google.registry.model.eppinput.EppInput; +import google.registry.model.eppoutput.EppOutput; +import google.registry.xml.ValidationMode; +import google.registry.xml.XmlException; +import google.registry.xml.XmlTransformer; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +/** {@link XmlTransformer} for marshalling to and from the Epp model classes. */ +public class EppXmlTransformer { + + // Hardcoded XML schemas, ordered with respect to dependency. + private static final ImmutableList SCHEMAS = ImmutableList.of( + "eppcom.xsd", + "epp.xsd", + "contact.xsd", + "host.xsd", + "domain.xsd", + "rgp.xsd", + "secdns.xsd", + "fee06.xsd", + "fee11.xsd", + "fee12.xsd", + "metadata.xsd", + "mark.xsd", + "dsig.xsd", + "smd.xsd", + "launch.xsd", + "allocate.xsd", + "superuser.xsd", + "allocationToken-1.0.xsd"); + + private static final XmlTransformer INPUT_TRANSFORMER = + new XmlTransformer(SCHEMAS, EppInput.class); + + private static final XmlTransformer OUTPUT_TRANSFORMER = + new XmlTransformer(SCHEMAS, EppOutput.class); + + public static void validateOutput(String xml) throws XmlException { + OUTPUT_TRANSFORMER.validate(xml); + } + + /** + * Unmarshal bytes into Epp classes. + * + * @param clazz type to return, specified as a param to enforce typesafe generics + */ + public static T unmarshal(Class clazz, byte[] bytes) throws XmlException { + return INPUT_TRANSFORMER.unmarshal(clazz, new ByteArrayInputStream(bytes)); + } + + private static byte[] marshal( + XmlTransformer transformer, + ImmutableObject root, + ValidationMode validation) throws XmlException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + transformer.marshal(root, byteArrayOutputStream, UTF_8, validation); + return byteArrayOutputStream.toByteArray(); + } + + public static byte[] marshal(EppOutput root, ValidationMode validation) throws XmlException { + return marshal(OUTPUT_TRANSFORMER, root, validation); + } + + @VisibleForTesting + public static byte[] marshalInput(EppInput root, ValidationMode validation) throws XmlException { + return marshal(INPUT_TRANSFORMER, root, validation); + } + + @VisibleForTesting + public static void validateInput(String xml) throws XmlException { + INPUT_TRANSFORMER.validate(xml); + } +} diff --git a/java/google/registry/model/index/DomainApplicationIndex.java b/java/google/registry/model/index/DomainApplicationIndex.java index f133fbaf3..7440df66f 100644 --- a/java/google/registry/model/index/DomainApplicationIndex.java +++ b/java/google/registry/model/index/DomainApplicationIndex.java @@ -90,7 +90,6 @@ public class DomainApplicationIndex extends BackupGroupRoot { *

Consequently within a transaction this method will not return any applications that are not * yet committed to datastore, even if called on an updated DomainApplicationIndex instance * storing keys to those applications. - * */ public static ImmutableSet loadActiveApplicationsByDomainName( String fullyQualifiedDomainName, final DateTime now) { diff --git a/java/google/registry/model/registrar/Registrar.java b/java/google/registry/model/registrar/Registrar.java index b67910cbe..23f026159 100644 --- a/java/google/registry/model/registrar/Registrar.java +++ b/java/google/registry/model/registrar/Registrar.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.ImmutableSortedMap.toImmutableSortedMap; import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet; import static com.google.common.collect.Ordering.natural; @@ -68,6 +69,7 @@ import google.registry.model.UpdateAutoTimestamp; import google.registry.model.annotations.ReportedOn; import google.registry.model.common.EntityGroupRoot; import google.registry.model.registrar.Registrar.BillingAccountEntry.CurrencyMapper; +import google.registry.model.registry.Registry; import google.registry.util.CidrAddressBlock; import google.registry.util.NonFinalForTesting; import java.security.MessageDigest; @@ -704,6 +706,28 @@ public class Registrar extends ImmutableObject implements Buildable, Jsonifiable return this; } + /** + * Same as {@link #setAllowedTlds}, but doesn't use the cache to check if the TLDs exist. + * + *

This should be used if the TLD we want to set is persisted in the same transaction - + * meaning its existence can't be cached before we need to save the Registrar. + * + *

We can still only set the allowedTld AFTER we saved the Registry entity. Make sure to call + * {@code .now()} when saving the Registry entity to make sure it's actually saved before trying + * to set the allowed TLDs. + */ + public Builder setAllowedTldsUncached(Set allowedTlds) { + ImmutableSet> newTldKeys = + Sets.difference(allowedTlds, getInstance().getAllowedTlds()).stream() + .map(tld -> Key.create(getCrossTldKey(), Registry.class, tld)) + .collect(toImmutableSet()); + Set> missingTldKeys = + Sets.difference(newTldKeys, ofy().load().keys(newTldKeys).keySet()); + checkArgument(missingTldKeys.isEmpty(), "Trying to set nonexisting TLDs: %s", missingTldKeys); + getInstance().allowedTlds = ImmutableSortedSet.copyOf(allowedTlds); + return this; + } + public Builder setClientCertificate(String clientCertificate, DateTime now) { clientCertificate = emptyToNull(clientCertificate); String clientCertificateHash = calculateHash(clientCertificate); diff --git a/java/google/registry/model/registry/label/PremiumList.java b/java/google/registry/model/registry/label/PremiumList.java index c4eb49d52..40b9b62b0 100644 --- a/java/google/registry/model/registry/label/PremiumList.java +++ b/java/google/registry/model/registry/label/PremiumList.java @@ -132,16 +132,22 @@ public final class PremiumList extends BaseDomainLabelListThis is cached for a shorter duration because we need to periodically reload this entity to * check if a new revision has been published, and if so, then use that. */ - static final LoadingCache cachePremiumLists = - CacheBuilder.newBuilder() - .expireAfterWrite(getDomainLabelListCacheDuration().getMillis(), MILLISECONDS) - .build( - new CacheLoader() { - @Override - public PremiumList load(final String name) { - return ofy().doTransactionless(() -> loadPremiumList(name)); - } - }); + @NonFinalForTesting + static LoadingCache cachePremiumLists = + createCachePremiumLists(getDomainLabelListCacheDuration()); + + @VisibleForTesting + static LoadingCache createCachePremiumLists(Duration cachePersistDuration) { + return CacheBuilder.newBuilder() + .expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS) + .build( + new CacheLoader() { + @Override + public PremiumList load(final String name) { + return ofy().doTransactionless(() -> loadPremiumList(name)); + } + }); + } private static PremiumList loadPremiumList(String name) { return ofy().load().type(PremiumList.class).parent(getCrossTldKey()).id(name).now(); diff --git a/java/google/registry/model/registry/label/PremiumListUtils.java b/java/google/registry/model/registry/label/PremiumListUtils.java index e9c1515d7..d8df27851 100644 --- a/java/google/registry/model/registry/label/PremiumListUtils.java +++ b/java/google/registry/model/registry/label/PremiumListUtils.java @@ -174,6 +174,12 @@ public final class PremiumListUtils { ofy().save().entities(newList, newRevision); return newList; }); + + // Invalidate the cache on this premium list so the change will take effect instantly. This only + // clears the cache on the same instance that the update was run on, which will typically be the + // only tools instance. + PremiumList.cachePremiumLists.invalidate(premiumList.getName()); + // TODO(b/79888775): Enqueue the oldPremiumList for deletion after at least // RegistryConfig.getDomainLabelListCacheDuration() has elapsed. return updated; diff --git a/java/google/registry/model/tmch/ClaimsListShard.java b/java/google/registry/model/tmch/ClaimsListShard.java index db507acdd..7527296b5 100644 --- a/java/google/registry/model/tmch/ClaimsListShard.java +++ b/java/google/registry/model/tmch/ClaimsListShard.java @@ -16,6 +16,7 @@ package google.registry.model.tmch; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.base.Verify.verify; import static google.registry.model.CacheUtils.memoizeWithShortExpiration; import static google.registry.model.ofy.ObjectifyService.allocateId; @@ -25,6 +26,7 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.UncheckedExecutionException; import com.googlecode.objectify.Key; import com.googlecode.objectify.annotation.EmbedMap; import com.googlecode.objectify.annotation.Entity; @@ -102,20 +104,31 @@ public class ClaimsListShard extends ImmutableObject { final List> shardKeys = ofy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list(); - // Load all of the shards concurrently, each in a separate transaction. - List shards = - Concurrent.transform( - shardKeys, - (final Key key) -> - ofy() - .transactNewReadOnly( - () -> { - ClaimsListShard claimsListShard = ofy().load().key(key).now(); - checkState( - claimsListShard != null, - "Key not found when loading claims list shards."); - return claimsListShard; - })); + List shards; + try { + // Load all of the shards concurrently, each in a separate transaction. + shards = + Concurrent.transform( + shardKeys, + key -> + ofy() + .transactNewReadOnly( + () -> { + ClaimsListShard claimsListShard = ofy().load().key(key).now(); + checkState( + claimsListShard != null, + "Key not found when loading claims list shards."); + return claimsListShard; + })); + } catch (UncheckedExecutionException e) { + // We retry on IllegalStateException. However, there's a checkState inside the + // Concurrent.transform, so if it's thrown it'll be wrapped in an + // UncheckedExecutionException. We want to unwrap it so it's caught by the retrier. + if (e.getCause() != null) { + throwIfUnchecked(e.getCause()); + } + throw e; + } // Combine the shards together and return the concatenated ClaimsList. if (!shards.isEmpty()) { diff --git a/java/google/registry/module/BUILD b/java/google/registry/module/BUILD new file mode 100644 index 000000000..84346bf7f --- /dev/null +++ b/java/google/registry/module/BUILD @@ -0,0 +1,23 @@ +package( + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +java_library( + name = "module", + srcs = glob(["*.java"]), + deps = [ + "//java/google/registry/request", + "//java/google/registry/util", + "@com_google_appengine_api_1_0_sdk", + "@com_google_dagger", + "@com_google_flogger", + "@com_google_flogger_system_backend", + "@com_google_monitoring_client_metrics", + "@javax_inject", + "@javax_servlet_api", + "@joda_time", + "@org_bouncycastle_bcpkix_jdk15on", + ], +) diff --git a/java/google/registry/module/ServletBase.java b/java/google/registry/module/ServletBase.java new file mode 100644 index 000000000..99b2a0dcc --- /dev/null +++ b/java/google/registry/module/ServletBase.java @@ -0,0 +1,83 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.module; + +import com.google.appengine.api.LifecycleManager; +import com.google.common.flogger.FluentLogger; +import com.google.monitoring.metrics.MetricReporter; +import dagger.Lazy; +import google.registry.request.RequestHandler; +import google.registry.util.SystemClock; +import java.io.IOException; +import java.security.Security; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.joda.time.DateTime; + +/** Base for Servlets that handle all requests to our App Engine modules. */ +public class ServletBase extends HttpServlet { + + private final RequestHandler requestHandler; + private final Lazy metricReporter; + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final SystemClock clock = new SystemClock(); + + public ServletBase(RequestHandler requestHandler, Lazy metricReporter) { + this.requestHandler = requestHandler; + this.metricReporter = metricReporter; + } + + @Override + public void init() { + Security.addProvider(new BouncyCastleProvider()); + + // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, + // etc), we log the error but keep the main thread running. Also the shutdown hook will only be + // registered if metric reporter starts up correctly. + try { + metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); + logger.atInfo().log("Started up MetricReporter"); + LifecycleManager.getInstance() + .setShutdownHook( + () -> { + try { + metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); + logger.atInfo().log("Shut down MetricReporter"); + } catch (TimeoutException e) { + logger.atSevere().withCause(e).log("Failed to stop MetricReporter."); + } + }); + } catch (Exception e) { + logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); + } + } + + @Override + public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { + logger.atInfo().log("Received %s request", getClass().getSimpleName()); + DateTime startTime = clock.nowUtc(); + try { + requestHandler.handleRequest(req, rsp); + } finally { + logger.atInfo().log( + "Finished %s request. Latency: %.3fs", + getClass().getSimpleName(), (clock.nowUtc().getMillis() - startTime.getMillis()) / 1000d); + } + } +} diff --git a/java/google/registry/module/backend/BUILD b/java/google/registry/module/backend/BUILD index 75aa46b18..547abad46 100644 --- a/java/google/registry/module/backend/BUILD +++ b/java/google/registry/module/backend/BUILD @@ -18,6 +18,7 @@ java_library( "//java/google/registry/dns/writer/clouddns", "//java/google/registry/dns/writer/dnsupdate", "//java/google/registry/export", + "//java/google/registry/export/datastore", "//java/google/registry/export/sheet", "//java/google/registry/flows", "//java/google/registry/gcs", @@ -27,6 +28,7 @@ java_library( "//java/google/registry/keyring/kms", "//java/google/registry/mapreduce", "//java/google/registry/model", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/rde", "//java/google/registry/rde/imports", diff --git a/java/google/registry/module/backend/BackendComponent.java b/java/google/registry/module/backend/BackendComponent.java index 9844eee40..2f792cd9c 100644 --- a/java/google/registry/module/backend/BackendComponent.java +++ b/java/google/registry/module/backend/BackendComponent.java @@ -22,6 +22,7 @@ import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.dns.writer.VoidDnsWriterModule; import google.registry.export.DriveModule; +import google.registry.export.datastore.DatastoreAdminModule; import google.registry.export.sheet.SheetsServiceModule; import google.registry.gcs.GcsServiceModule; import google.registry.groups.DirectoryModule; @@ -56,6 +57,7 @@ import javax.inject.Singleton; BigqueryModule.class, ConfigModule.class, CredentialModule.class, + DatastoreAdminModule.class, DatastoreServiceModule.class, DirectoryModule.class, DummyKeyringModule.class, diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index 09c7fa8ab..f1236d4f4 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -39,7 +39,9 @@ import google.registry.dns.writer.VoidDnsWriterModule; import google.registry.dns.writer.clouddns.CloudDnsWriterModule; import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule; import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule; +import google.registry.export.BackupDatastoreAction; import google.registry.export.BigqueryPollJobAction; +import google.registry.export.CheckBackupAction; import google.registry.export.CheckSnapshotAction; import google.registry.export.ExportDomainListsAction; import google.registry.export.ExportPremiumTermsAction; @@ -49,6 +51,7 @@ import google.registry.export.ExportSnapshotAction; import google.registry.export.LoadSnapshotAction; import google.registry.export.SyncGroupMembersAction; import google.registry.export.UpdateSnapshotViewAction; +import google.registry.export.UploadDatastoreBackupAction; import google.registry.export.sheet.SheetModule; import google.registry.export.sheet.SyncRegistrarsSheetAction; import google.registry.flows.async.AsyncFlowsModule; @@ -70,6 +73,7 @@ import google.registry.reporting.billing.BillingModule; import google.registry.reporting.billing.CopyDetailReportsAction; import google.registry.reporting.billing.GenerateInvoicesAction; import google.registry.reporting.billing.PublishInvoicesAction; +import google.registry.reporting.icann.DnsCountQueryCoordinatorModule; import google.registry.reporting.icann.IcannReportingModule; import google.registry.reporting.icann.IcannReportingStagingAction; import google.registry.reporting.icann.IcannReportingUploadAction; @@ -97,6 +101,7 @@ import google.registry.tmch.TmchSmdrlAction; BillingModule.class, CloudDnsWriterModule.class, CronModule.class, + DnsCountQueryCoordinatorModule.class, DnsModule.class, DnsUpdateConfigModule.class, DnsUpdateWriterModule.class, @@ -114,8 +119,10 @@ import google.registry.tmch.TmchSmdrlAction; WhiteboxModule.class, }) interface BackendRequestComponent { + BackupDatastoreAction backupDatastoreAction(); BigqueryPollJobAction bigqueryPollJobAction(); BrdaCopyAction brdaCopyAction(); + CheckBackupAction checkBackupAction(); CheckSnapshotAction checkSnapshotAction(); CommitLogCheckpointAction commitLogCheckpointAction(); CommitLogFanoutAction commitLogFanoutAction(); @@ -158,6 +165,7 @@ interface BackendRequestComponent { TmchCrlAction tmchCrlAction(); TmchDnlAction tmchDnlAction(); TmchSmdrlAction tmchSmdrlAction(); + UploadDatastoreBackupAction uploadDatastoreBackupAction(); UpdateSnapshotViewAction updateSnapshotViewAction(); PublishInvoicesAction uploadInvoicesAction(); diff --git a/java/google/registry/module/backend/BackendServlet.java b/java/google/registry/module/backend/BackendServlet.java index 328775f13..896840937 100644 --- a/java/google/registry/module/backend/BackendServlet.java +++ b/java/google/registry/module/backend/BackendServlet.java @@ -14,56 +14,18 @@ package google.registry.module.backend; -import com.google.appengine.api.LifecycleManager; -import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; -import java.io.IOException; -import java.security.Security; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "backend" App Engine module. */ -public final class BackendServlet extends HttpServlet { +public final class BackendServlet extends ServletBase { private static final BackendComponent component = DaggerBackendComponent.create(); private static final BackendRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - - // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, - // etc), we log the error but keep the main thread running. Also the shutdown hook will only be - // registered if metric reporter starts up correctly. - try { - metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); - logger.atInfo().log("Started up MetricReporter"); - LifecycleManager.getInstance() - .setShutdownHook( - () -> { - try { - metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); - logger.atInfo().log("Shut down MetricReporter"); - } catch (TimeoutException timeoutException) { - logger.atSevere().withCause(timeoutException).log( - "Failed to stop MetricReporter: %s", timeoutException); - } - }); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); - } - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received backend request"); - requestHandler.handleRequest(req, rsp); + public BackendServlet() { + super(requestHandler, metricReporter); } } diff --git a/java/google/registry/module/frontend/BUILD b/java/google/registry/module/frontend/BUILD index 589a82287..fc2d681fc 100644 --- a/java/google/registry/module/frontend/BUILD +++ b/java/google/registry/module/frontend/BUILD @@ -11,14 +11,17 @@ java_library( "//java/google/registry/config", "//java/google/registry/dns", "//java/google/registry/flows", + "//java/google/registry/groups", "//java/google/registry/keyring", "//java/google/registry/keyring/api", "//java/google/registry/keyring/kms", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/request", "//java/google/registry/request:modules", "//java/google/registry/request/auth", "//java/google/registry/ui", + "//java/google/registry/ui/server/otesetup", "//java/google/registry/ui/server/registrar", "//java/google/registry/util", "@com_google_appengine_api_1_0_sdk", @@ -29,6 +32,7 @@ java_library( "@com_google_monitoring_client_metrics", "@javax_inject", "@javax_servlet_api", + "@joda_time", "@org_bouncycastle_bcpkix_jdk15on", ], ) diff --git a/java/google/registry/module/frontend/FrontendComponent.java b/java/google/registry/module/frontend/FrontendComponent.java index 1c96edc5b..04314dbe2 100644 --- a/java/google/registry/module/frontend/FrontendComponent.java +++ b/java/google/registry/module/frontend/FrontendComponent.java @@ -21,6 +21,9 @@ import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; +import google.registry.groups.DirectoryModule; +import google.registry.groups.GroupsModule; +import google.registry.groups.GroupssettingsModule; import google.registry.keyring.KeyringModule; import google.registry.keyring.api.DummyKeyringModule; import google.registry.keyring.api.KeyModule; @@ -48,8 +51,11 @@ import javax.inject.Singleton; ConsoleConfigModule.class, CredentialModule.class, CustomLogicFactoryModule.class, + DirectoryModule.class, DummyKeyringModule.class, FrontendRequestComponentModule.class, + GroupsModule.class, + GroupssettingsModule.class, Jackson2Module.class, KeyModule.class, KeyringModule.class, diff --git a/java/google/registry/module/frontend/FrontendRequestComponent.java b/java/google/registry/module/frontend/FrontendRequestComponent.java index 19c22964b..277f2d745 100644 --- a/java/google/registry/module/frontend/FrontendRequestComponent.java +++ b/java/google/registry/module/frontend/FrontendRequestComponent.java @@ -25,7 +25,9 @@ import google.registry.monitoring.whitebox.WhiteboxModule; import google.registry.request.RequestComponentBuilder; import google.registry.request.RequestModule; import google.registry.request.RequestScope; +import google.registry.ui.server.otesetup.ConsoleOteSetupAction; import google.registry.ui.server.registrar.ConsoleUiAction; +import google.registry.ui.server.registrar.OteStatusAction; import google.registry.ui.server.registrar.RegistrarConsoleModule; import google.registry.ui.server.registrar.RegistrarSettingsAction; @@ -33,17 +35,19 @@ import google.registry.ui.server.registrar.RegistrarSettingsAction; @RequestScope @Subcomponent( modules = { - RegistrarConsoleModule.class, DnsModule.class, EppTlsModule.class, + RegistrarConsoleModule.class, RequestModule.class, WhiteboxModule.class, }) interface FrontendRequestComponent { + ConsoleOteSetupAction consoleOteSetupAction(); ConsoleUiAction consoleUiAction(); EppConsoleAction eppConsoleAction(); EppTlsAction eppTlsAction(); FlowComponent.Builder flowComponentBuilder(); + OteStatusAction oteStatusAction(); RegistrarSettingsAction registrarSettingsAction(); @Subcomponent.Builder diff --git a/java/google/registry/module/frontend/FrontendServlet.java b/java/google/registry/module/frontend/FrontendServlet.java index 455c410e5..2b2b72350 100644 --- a/java/google/registry/module/frontend/FrontendServlet.java +++ b/java/google/registry/module/frontend/FrontendServlet.java @@ -14,55 +14,18 @@ package google.registry.module.frontend; -import com.google.appengine.api.LifecycleManager; -import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; -import java.io.IOException; -import java.security.Security; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "default" App Engine module. */ -public final class FrontendServlet extends HttpServlet { +public final class FrontendServlet extends ServletBase { private static final FrontendComponent component = DaggerFrontendComponent.create(); private static final FrontendRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - - // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, - // etc), we log the error but keep the main thread running. Also the shutdown hook will only be - // registered if metric reporter starts up correctly. - try { - metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); - logger.atInfo().log("Started up MetricReporter"); - LifecycleManager.getInstance() - .setShutdownHook( - () -> { - try { - metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); - logger.atInfo().log("Shut down MetricReporter"); - } catch (TimeoutException e) { - logger.atSevere().withCause(e).log("Failed to stop MetricReporter."); - } - }); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); - } - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received frontend request"); - requestHandler.handleRequest(req, rsp); + public FrontendServlet() { + super(requestHandler, metricReporter); } } diff --git a/java/google/registry/module/pubapi/BUILD b/java/google/registry/module/pubapi/BUILD index e3d388311..5d0307c85 100644 --- a/java/google/registry/module/pubapi/BUILD +++ b/java/google/registry/module/pubapi/BUILD @@ -11,9 +11,11 @@ java_library( "//java/google/registry/config", "//java/google/registry/dns", "//java/google/registry/flows", + "//java/google/registry/groups", "//java/google/registry/keyring", "//java/google/registry/keyring/api", "//java/google/registry/keyring/kms", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/rdap", "//java/google/registry/request", @@ -29,6 +31,7 @@ java_library( "@com_google_monitoring_client_metrics", "@javax_inject", "@javax_servlet_api", + "@joda_time", "@org_bouncycastle_bcpkix_jdk15on", ], ) diff --git a/java/google/registry/module/pubapi/PubApiComponent.java b/java/google/registry/module/pubapi/PubApiComponent.java index ef5ffbc65..1b65390a0 100644 --- a/java/google/registry/module/pubapi/PubApiComponent.java +++ b/java/google/registry/module/pubapi/PubApiComponent.java @@ -21,6 +21,9 @@ import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; +import google.registry.groups.DirectoryModule; +import google.registry.groups.GroupsModule; +import google.registry.groups.GroupssettingsModule; import google.registry.keyring.KeyringModule; import google.registry.keyring.api.DummyKeyringModule; import google.registry.keyring.api.KeyModule; @@ -46,13 +49,16 @@ import javax.inject.Singleton; ConfigModule.class, CredentialModule.class, CustomLogicFactoryModule.class, + DirectoryModule.class, DummyKeyringModule.class, - PubApiRequestComponentModule.class, + GroupsModule.class, + GroupssettingsModule.class, Jackson2Module.class, KeyModule.class, KeyringModule.class, KmsModule.class, NetHttpTransportModule.class, + PubApiRequestComponentModule.class, ServerTridProviderModule.class, StackdriverModule.class, SystemClockModule.class, diff --git a/java/google/registry/module/pubapi/PubApiServlet.java b/java/google/registry/module/pubapi/PubApiServlet.java index 15d48d9a9..b907f9042 100644 --- a/java/google/registry/module/pubapi/PubApiServlet.java +++ b/java/google/registry/module/pubapi/PubApiServlet.java @@ -14,55 +14,18 @@ package google.registry.module.pubapi; -import com.google.appengine.api.LifecycleManager; -import com.google.common.flogger.FluentLogger; import com.google.monitoring.metrics.MetricReporter; import dagger.Lazy; -import java.io.IOException; -import java.security.Security; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "default" App Engine module. */ -public final class PubApiServlet extends HttpServlet { +public final class PubApiServlet extends ServletBase { private static final PubApiComponent component = DaggerPubApiComponent.create(); private static final PubApiRequestHandler requestHandler = component.requestHandler(); private static final Lazy metricReporter = component.metricReporter(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - - // If metric reporter failed to instantiate for any reason (bad keyring, bad json credential, - // etc), we log the error but keep the main thread running. Also the shutdown hook will only be - // registered if metric reporter starts up correctly. - try { - metricReporter.get().startAsync().awaitRunning(10, TimeUnit.SECONDS); - logger.atInfo().log("Started up MetricReporter"); - LifecycleManager.getInstance() - .setShutdownHook( - () -> { - try { - metricReporter.get().stopAsync().awaitTerminated(10, TimeUnit.SECONDS); - logger.atInfo().log("Shut down MetricReporter"); - } catch (TimeoutException e) { - logger.atSevere().withCause(e).log("Failed to stop MetricReporter."); - } - }); - } catch (Exception e) { - logger.atSevere().withCause(e).log("Failed to initialize MetricReporter."); - } - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received frontend request"); - requestHandler.handleRequest(req, rsp); + public PubApiServlet() { + super(requestHandler, metricReporter); } } diff --git a/java/google/registry/module/tools/BUILD b/java/google/registry/module/tools/BUILD index a46e09e90..39d5dc545 100644 --- a/java/google/registry/module/tools/BUILD +++ b/java/google/registry/module/tools/BUILD @@ -20,6 +20,7 @@ java_library( "//java/google/registry/keyring/kms", "//java/google/registry/loadtest", "//java/google/registry/mapreduce", + "//java/google/registry/module", "//java/google/registry/monitoring/whitebox", "//java/google/registry/request", "//java/google/registry/request:modules", @@ -30,8 +31,10 @@ java_library( "@com_google_dagger", "@com_google_flogger", "@com_google_flogger_system_backend", + "@com_google_monitoring_client_metrics", "@javax_inject", "@javax_servlet_api", + "@joda_time", "@org_bouncycastle_bcpkix_jdk15on", ], ) diff --git a/java/google/registry/module/tools/ToolsComponent.java b/java/google/registry/module/tools/ToolsComponent.java index 605ddc95a..b7dc1747a 100644 --- a/java/google/registry/module/tools/ToolsComponent.java +++ b/java/google/registry/module/tools/ToolsComponent.java @@ -14,7 +14,9 @@ package google.registry.module.tools; +import com.google.monitoring.metrics.MetricReporter; import dagger.Component; +import dagger.Lazy; import google.registry.config.CredentialModule; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.export.DriveModule; @@ -29,6 +31,7 @@ import google.registry.keyring.api.DummyKeyringModule; import google.registry.keyring.api.KeyModule; import google.registry.keyring.kms.KmsModule; import google.registry.module.tools.ToolsRequestComponent.ToolsRequestComponentModule; +import google.registry.monitoring.whitebox.StackdriverModule; import google.registry.request.Modules.DatastoreServiceModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.NetHttpTransportModule; @@ -62,6 +65,7 @@ import javax.inject.Singleton; KmsModule.class, NetHttpTransportModule.class, ServerTridProviderModule.class, + StackdriverModule.class, SystemClockModule.class, SystemSleeperModule.class, ToolsRequestComponentModule.class, @@ -70,4 +74,6 @@ import javax.inject.Singleton; }) interface ToolsComponent { ToolsRequestHandler requestHandler(); + + Lazy metricReporter(); } diff --git a/java/google/registry/module/tools/ToolsRequestComponent.java b/java/google/registry/module/tools/ToolsRequestComponent.java index 633a38709..e8e6879c5 100644 --- a/java/google/registry/module/tools/ToolsRequestComponent.java +++ b/java/google/registry/module/tools/ToolsRequestComponent.java @@ -34,6 +34,7 @@ import google.registry.tools.server.CreatePremiumListAction; import google.registry.tools.server.DeleteEntityAction; import google.registry.tools.server.GenerateZoneFilesAction; import google.registry.tools.server.KillAllCommitLogsAction; +import google.registry.tools.server.KillAllDomainApplicationsAction; import google.registry.tools.server.KillAllEppResourcesAction; import google.registry.tools.server.ListDomainsAction; import google.registry.tools.server.ListHostsAction; @@ -69,6 +70,7 @@ interface ToolsRequestComponent { FlowComponent.Builder flowComponentBuilder(); GenerateZoneFilesAction generateZoneFilesAction(); KillAllCommitLogsAction killAllCommitLogsAction(); + KillAllDomainApplicationsAction killAllDomainApplicationsAction(); KillAllEppResourcesAction killAllEppResourcesAction(); ListDomainsAction listDomainsAction(); ListHostsAction listHostsAction(); diff --git a/java/google/registry/module/tools/ToolsServlet.java b/java/google/registry/module/tools/ToolsServlet.java index 79dd085b6..023c51164 100644 --- a/java/google/registry/module/tools/ToolsServlet.java +++ b/java/google/registry/module/tools/ToolsServlet.java @@ -14,29 +14,18 @@ package google.registry.module.tools; -import com.google.common.flogger.FluentLogger; -import java.io.IOException; -import java.security.Security; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.bouncycastle.jce.provider.BouncyCastleProvider; +import com.google.monitoring.metrics.MetricReporter; +import dagger.Lazy; +import google.registry.module.ServletBase; /** Servlet that should handle all requests to our "tools" App Engine module. */ -public final class ToolsServlet extends HttpServlet { +public final class ToolsServlet extends ServletBase { private static final ToolsComponent component = DaggerToolsComponent.create(); private static final ToolsRequestHandler requestHandler = component.requestHandler(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final Lazy metricReporter = component.metricReporter(); - @Override - public void init() { - Security.addProvider(new BouncyCastleProvider()); - } - - @Override - public void service(HttpServletRequest req, HttpServletResponse rsp) throws IOException { - logger.atInfo().log("Received tools request"); - requestHandler.handleRequest(req, rsp); + public ToolsServlet() { + super(requestHandler, metricReporter); } } diff --git a/java/google/registry/proxy/BUILD b/java/google/registry/proxy/BUILD index 8c9415278..2d02ee8f3 100644 --- a/java/google/registry/proxy/BUILD +++ b/java/google/registry/proxy/BUILD @@ -18,18 +18,15 @@ java_library( "config/*.yaml", ]), deps = [ - "//java/google/registry/config", "//java/google/registry/util", "@com_beust_jcommander", - "@com_fasterxml_jackson_core", - "@com_fasterxml_jackson_core_jackson_annotations", - "@com_fasterxml_jackson_core_jackson_databind", "@com_google_api_client", "@com_google_apis_google_api_services_cloudkms", "@com_google_apis_google_api_services_monitoring", "@com_google_apis_google_api_services_storage", "@com_google_auto_value", "@com_google_code_findbugs_jsr305", + "@com_google_code_gson", "@com_google_dagger", "@com_google_flogger", "@com_google_flogger_system_backend", diff --git a/java/google/registry/proxy/GcpJsonFormatter.java b/java/google/registry/proxy/GcpJsonFormatter.java index e7b2d391f..7fd3c0a53 100644 --- a/java/google/registry/proxy/GcpJsonFormatter.java +++ b/java/google/registry/proxy/GcpJsonFormatter.java @@ -14,10 +14,8 @@ package google.registry.proxy; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; import java.io.PrintWriter; import java.io.StringWriter; import java.util.logging.Formatter; @@ -41,98 +39,89 @@ import java.util.logging.LogRecord; */ class GcpJsonFormatter extends Formatter { - private static final ObjectMapper MAPPER = new ObjectMapper(); + /** JSON field that determines the log level. */ + private static final String SEVERITY = "severity"; + + /** + * JSON field that stores the calling class and function when the log occurs. + * + *

This field is not used by Stackdriver, but it is useful and can be found when the log + * entries are expanded + */ + private static final String SOURCE = "source"; + + /** JSON field that contains the content, this will show up as the main entry in a log. */ + private static final String MESSAGE = "message"; + + private static final Gson gson = new Gson(); @Override public String format(LogRecord record) { - try { - return MAPPER.writeValueAsString(LogEvent.create(record)) + "\n"; - } catch (JsonProcessingException e) { - throw new RuntimeException(e); + // Add an extra newline before the message. Stackdriver does not show newlines correctly, and + // treats them as whitespace. If you want to see correctly formatted log message, expand the + // log and look for the jsonPayload.message field. This newline makes sure that the entire + // message starts on its own line, so that indentation within the message is correct. + + String message = "\n" + record.getMessage(); + String severity = severityFor(record.getLevel()); + + // The rest is mostly lifted from java.util.logging.SimpleFormatter. + String stacktrace = ""; + if (record.getThrown() != null) { + StringWriter sw = new StringWriter(); + try (PrintWriter pw = new PrintWriter(sw)) { + pw.println(); + record.getThrown().printStackTrace(pw); + } + stacktrace = sw.toString(); } + + String source; + if (record.getSourceClassName() != null) { + source = record.getSourceClassName(); + if (record.getSourceMethodName() != null) { + source += " " + record.getSourceMethodName(); + } + } else { + source = record.getLoggerName(); + } + + return gson.toJson( + ImmutableMap.of(SEVERITY, severity, SOURCE, source, MESSAGE, message + stacktrace)) + + '\n'; } - @AutoValue - abstract static class LogEvent { - - /** Field that determines the log level. */ - @JsonProperty("severity") - abstract String severity(); - - /** - * Field that stores the calling class and function when the log occurs. - * - *

This field is not used by Stackdriver, but it is useful and can be found when the log - * entries are expanded - */ - @JsonProperty("source") - abstract String source(); - - /** Field that contains the content, this will show up as the main entry in a log. */ - @JsonProperty("message") - abstract String message(); - - static LogEvent create(LogRecord record) { - // Add an extra newline before the message. Stackdriver does not show newlines correctly, and - // treats them as whitespace. If you want to see correctly formatted log message, expand the - // log and look for the jsonPayload.message field. This newline makes sure that the entire - // message starts on its own line, so that indentation within the message is correct. - - String message = "\n" + record.getMessage(); - Level level = record.getLevel(); - // See - // https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/google-cloud-logging/src/main/java/com/google/cloud/logging/Severity.java - // on how {@code Level} is mapped to severity. - String severity; - switch (level.intValue()) { - // FINEST - case 300: - // FINER - case 400: - // FINE - case 500: - severity = "DEBUG"; - break; - // CONFIG - case 700: - // INFO - case 800: - severity = "INFO"; - break; - // WARNING - case 900: - severity = "WARNING"; - break; - // SEVERE - case 1000: - severity = "ERROR"; - break; - default: - severity = "DEFAULT"; - } - - // The rest is mostly lifted from java.util.logging.SimpleFormatter. - String stacktrace = ""; - if (record.getThrown() != null) { - StringWriter sw = new StringWriter(); - try (PrintWriter pw = new PrintWriter(sw)) { - pw.println(); - record.getThrown().printStackTrace(pw); - } - stacktrace = sw.toString(); - } - - String source; - if (record.getSourceClassName() != null) { - source = record.getSourceClassName(); - if (record.getSourceMethodName() != null) { - source += " " + record.getSourceMethodName(); - } - } else { - source = record.getLoggerName(); - } - - return new AutoValue_GcpJsonFormatter_LogEvent(severity, source, message + stacktrace); + /** + * Map {@link Level} to a severity string that Stackdriver understands. + * + * @see {@code LoggingHandler} + */ + private static String severityFor(Level level) { + switch (level.intValue()) { + // FINEST + case 300: + return "DEBUG"; + // FINER + case 400: + return "DEBUG"; + // FINE + case 500: + return "DEBUG"; + // CONFIG + case 700: + return "INFO"; + // INFO + case 800: + return "INFO"; + // WARNING + case 900: + return "WARNING"; + // SEVERE + case 1000: + return "ERROR"; + default: + return "DEFAULT"; } } } diff --git a/java/google/registry/proxy/ProxyConfig.java b/java/google/registry/proxy/ProxyConfig.java index c23f3342b..f4fa9bd12 100644 --- a/java/google/registry/proxy/ProxyConfig.java +++ b/java/google/registry/proxy/ProxyConfig.java @@ -14,8 +14,8 @@ package google.registry.proxy; -import static google.registry.config.YamlUtils.getConfigSettings; import static google.registry.util.ResourceUtils.readResourceUtf8; +import static google.registry.util.YamlUtils.getConfigSettings; import com.google.common.base.Ascii; import java.util.List; diff --git a/java/google/registry/proxy/handler/ProxyProtocolHandler.java b/java/google/registry/proxy/handler/ProxyProtocolHandler.java index f86760658..2612f14d3 100644 --- a/java/google/registry/proxy/handler/ProxyProtocolHandler.java +++ b/java/google/registry/proxy/handler/ProxyProtocolHandler.java @@ -81,6 +81,15 @@ public class ProxyProtocolHandler extends ByteToMessageDecoder { remoteIP = headerArray[2]; logger.atFine().log( "Header parsed, using %s as remote IP for channel %s", remoteIP, ctx.channel()); + // If the header is "PROXY UNKNOWN" + // (see https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt), likely when the + // remote connection to the external load balancer is through special means, make it + // 0.0.0.0 so that it can be treated accordingly by the relevant quota configs. + } else if (headerArray.length == 2 && headerArray[1].equals("UNKNOWN")) { + logger.atFine().log( + "Header parsed, source IP unknown, using 0.0.0.0 as remote IP for channel %s", + ctx.channel()); + remoteIP = "0.0.0.0"; } else { logger.atFine().log( "Cannot parse the header, using source IP as remote IP for channel %s", diff --git a/java/google/registry/rdap/RdapActionBase.java b/java/google/registry/rdap/RdapActionBase.java index d1f448559..d0a313d90 100644 --- a/java/google/registry/rdap/RdapActionBase.java +++ b/java/google/registry/rdap/RdapActionBase.java @@ -47,8 +47,8 @@ import google.registry.request.RequestMethod; import google.registry.request.RequestPath; import google.registry.request.Response; import google.registry.request.auth.AuthResult; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; import google.registry.request.auth.UserAuthInfo; -import google.registry.ui.server.registrar.AuthenticatedRegistrarAccessor; import google.registry.util.Clock; import java.io.IOException; import java.net.URI; @@ -272,6 +272,18 @@ public abstract class RdapActionBase implements Runnable { || registrarParam.get().equals(eppResource.getPersistedCurrentSponsorClientId())); } + /** + * Returns true if the EPP resource should be visible. + * + *

This is true iff: + * 1. The passed in resource exists and is not deleted (deleted ones will have been projected + * forward in time to empty), + * 2. The request did not specify a registrar to filter on, or the registrar matches. + */ + boolean shouldBeVisible(Optional eppResource, DateTime now) { + return eppResource.isPresent() && shouldBeVisible(eppResource.get(), now); + } + /** * Returns true if the registrar should be visible. * diff --git a/java/google/registry/rdap/RdapDomainAction.java b/java/google/registry/rdap/RdapDomainAction.java index 2563a3987..2d975f112 100644 --- a/java/google/registry/rdap/RdapDomainAction.java +++ b/java/google/registry/rdap/RdapDomainAction.java @@ -29,6 +29,7 @@ import google.registry.request.Action; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.NotFoundException; import google.registry.request.auth.Auth; +import java.util.Optional; import javax.inject.Inject; import org.joda.time.DateTime; @@ -74,14 +75,14 @@ public class RdapDomainAction extends RdapActionBase { pathSearchString, getHumanReadableObjectTypeName(), e.getMessage())); } // The query string is not used; the RDAP syntax is /rdap/domain/mydomain.com. - DomainResource domainResource = + Optional domainResource = loadByForeignKey( DomainResource.class, pathSearchString, shouldIncludeDeleted() ? START_OF_TIME : now); - if ((domainResource == null) || !shouldBeVisible(domainResource, now)) { + if (!shouldBeVisible(domainResource, now)) { throw new NotFoundException(pathSearchString + " not found"); } return rdapJsonFormatter.makeRdapJsonForDomain( - domainResource, + domainResource.get(), true, fullServletPath, rdapWhoisServer, diff --git a/java/google/registry/rdap/RdapDomainSearchAction.java b/java/google/registry/rdap/RdapDomainSearchAction.java index 59c5ffac4..3999f6660 100644 --- a/java/google/registry/rdap/RdapDomainSearchAction.java +++ b/java/google/registry/rdap/RdapDomainSearchAction.java @@ -216,13 +216,13 @@ public class RdapDomainSearchAction extends RdapSearchActionBase { */ private RdapSearchResults searchByDomainNameWithoutWildcard( final RdapSearchPattern partialStringQuery, final DateTime now) { - DomainResource domainResource = + Optional domainResource = loadByForeignKey(DomainResource.class, partialStringQuery.getInitialString(), now); - ImmutableList results = - ((domainResource == null) || !shouldBeVisible(domainResource, now)) - ? ImmutableList.of() - : ImmutableList.of(domainResource); - return makeSearchResults(results, now); + return makeSearchResults( + shouldBeVisible(domainResource, now) + ? ImmutableList.of(domainResource.get()) + : ImmutableList.of(), + now); } /** Searches for domains by domain name with an initial string, wildcard and possible suffix. */ @@ -343,15 +343,15 @@ public class RdapDomainSearchAction extends RdapSearchActionBase { // the key. Optional desiredRegistrar = getDesiredRegistrar(); if (desiredRegistrar.isPresent()) { - HostResource host = + Optional host = loadByForeignKey( HostResource.class, partialStringQuery.getInitialString(), shouldIncludeDeleted() ? START_OF_TIME : now); - return ((host == null) - || !desiredRegistrar.get().equals(host.getPersistedCurrentSponsorClientId())) + return (!host.isPresent() + || !desiredRegistrar.get().equals(host.get().getPersistedCurrentSponsorClientId())) ? ImmutableList.of() - : ImmutableList.of(Key.create(host)); + : ImmutableList.of(Key.create(host.get())); } else { Key hostKey = loadAndGetKey( @@ -370,15 +370,14 @@ public class RdapDomainSearchAction extends RdapSearchActionBase { // with no initial string. DomainResource domainResource = loadByForeignKey( - DomainResource.class, - partialStringQuery.getSuffix(), - shouldIncludeDeleted() ? START_OF_TIME : now); - if (domainResource == null) { - // Don't allow wildcards with suffixes which are not domains we manage. That would risk a - // table scan in some easily foreseeable cases. - throw new UnprocessableEntityException( - "A suffix in a lookup by nameserver name must be a domain defined in the system"); - } + DomainResource.class, + partialStringQuery.getSuffix(), + shouldIncludeDeleted() ? START_OF_TIME : now) + .orElseThrow( + () -> + new UnprocessableEntityException( + "A suffix in a lookup by nameserver name " + + "must be a domain defined in the system")); Optional desiredRegistrar = getDesiredRegistrar(); ImmutableList.Builder> builder = new ImmutableList.Builder<>(); for (String fqhn : ImmutableSortedSet.copyOf(domainResource.getSubordinateHosts())) { @@ -386,12 +385,12 @@ public class RdapDomainSearchAction extends RdapSearchActionBase { // then the query ns.exam*.example.com would match against nameserver ns.example.com. if (partialStringQuery.matches(fqhn)) { if (desiredRegistrar.isPresent()) { - HostResource host = + Optional host = loadByForeignKey( HostResource.class, fqhn, shouldIncludeDeleted() ? START_OF_TIME : now); - if ((host != null) - && desiredRegistrar.get().equals(host.getPersistedCurrentSponsorClientId())) { - builder.add(Key.create(host)); + if (host.isPresent() + && desiredRegistrar.get().equals(host.get().getPersistedCurrentSponsorClientId())) { + builder.add(Key.create(host.get())); } } else { Key hostKey = diff --git a/java/google/registry/rdap/RdapNameserverAction.java b/java/google/registry/rdap/RdapNameserverAction.java index 27e7e44f1..4c898169d 100644 --- a/java/google/registry/rdap/RdapNameserverAction.java +++ b/java/google/registry/rdap/RdapNameserverAction.java @@ -29,6 +29,7 @@ import google.registry.request.Action; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.NotFoundException; import google.registry.request.auth.Auth; +import java.util.Optional; import javax.inject.Inject; import org.joda.time.DateTime; @@ -76,13 +77,13 @@ public class RdapNameserverAction extends RdapActionBase { } // If there are no undeleted nameservers with the given name, the foreign key should point to // the most recently deleted one. - HostResource hostResource = + Optional hostResource = loadByForeignKey( HostResource.class, pathSearchString, shouldIncludeDeleted() ? START_OF_TIME : now); - if ((hostResource == null) || !shouldBeVisible(hostResource, now)) { + if (!shouldBeVisible(hostResource, now)) { throw new NotFoundException(pathSearchString + " not found"); } return rdapJsonFormatter.makeRdapJsonForHost( - hostResource, true, fullServletPath, rdapWhoisServer, now, OutputDataType.FULL); + hostResource.get(), true, fullServletPath, rdapWhoisServer, now, OutputDataType.FULL); } } diff --git a/java/google/registry/rdap/RdapNameserverSearchAction.java b/java/google/registry/rdap/RdapNameserverSearchAction.java index b36eddda5..b77909d05 100644 --- a/java/google/registry/rdap/RdapNameserverSearchAction.java +++ b/java/google/registry/rdap/RdapNameserverSearchAction.java @@ -183,9 +183,9 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { */ private RdapSearchResults searchByNameUsingForeignKey( final RdapSearchPattern partialStringQuery, final DateTime now) { - HostResource hostResource = + Optional hostResource = loadByForeignKey(HostResource.class, partialStringQuery.getInitialString(), now); - if ((hostResource == null) || !shouldBeVisible(hostResource, now)) { + if (!shouldBeVisible(hostResource, now)) { metricInformationBuilder.setNumHostsRetrieved(0); throw new NotFoundException("No nameservers found"); } @@ -193,15 +193,20 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { return RdapSearchResults.create( ImmutableList.of( rdapJsonFormatter.makeRdapJsonForHost( - hostResource, false, fullServletPath, rdapWhoisServer, now, OutputDataType.FULL))); + hostResource.get(), + false, + fullServletPath, + rdapWhoisServer, + now, + OutputDataType.FULL))); } /** Searches for nameservers by name using the superordinate domain as a suffix. */ private RdapSearchResults searchByNameUsingSuperordinateDomain( final RdapSearchPattern partialStringQuery, final DateTime now) { - DomainResource domainResource = + Optional domainResource = loadByForeignKey(DomainResource.class, partialStringQuery.getSuffix(), now); - if (domainResource == null) { + if (!domainResource.isPresent()) { // Don't allow wildcards with suffixes which are not domains we manage. That would risk a // table scan in many easily foreseeable cases. The user might ask for ns*.zombo.com, // forcing us to query for all hosts beginning with ns, then filter for those ending in @@ -211,16 +216,16 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { "A suffix after a wildcard in a nameserver lookup must be an in-bailiwick domain"); } List hostList = new ArrayList<>(); - for (String fqhn : ImmutableSortedSet.copyOf(domainResource.getSubordinateHosts())) { + for (String fqhn : ImmutableSortedSet.copyOf(domainResource.get().getSubordinateHosts())) { if (cursorString.isPresent() && (fqhn.compareTo(cursorString.get()) <= 0)) { continue; } // We can't just check that the host name starts with the initial query string, because // then the query ns.exam*.example.com would match against nameserver ns.example.com. if (partialStringQuery.matches(fqhn)) { - HostResource hostResource = loadByForeignKey(HostResource.class, fqhn, now); - if ((hostResource != null) && shouldBeVisible(hostResource, now)) { - hostList.add(hostResource); + Optional hostResource = loadByForeignKey(HostResource.class, fqhn, now); + if (shouldBeVisible(hostResource, now)) { + hostList.add(hostResource.get()); if (hostList.size() > rdapResultSetMaxSize) { break; } @@ -230,7 +235,7 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase { return makeSearchResults( hostList, IncompletenessWarningType.COMPLETE, - domainResource.getSubordinateHosts().size(), + domainResource.get().getSubordinateHosts().size(), CursorType.NAME, now); } diff --git a/java/google/registry/rde/imports/RdeHostLinkAction.java b/java/google/registry/rde/imports/RdeHostLinkAction.java index e8f8ee147..8b81f826f 100644 --- a/java/google/registry/rde/imports/RdeHostLinkAction.java +++ b/java/google/registry/rde/imports/RdeHostLinkAction.java @@ -195,11 +195,14 @@ public class RdeHostLinkAction implements Runnable { .stream() .skip(hostName.parts().size() - (tld.get().parts().size() + 1)) .collect(joining(".")); - DomainResource superordinateDomain = loadByForeignKey(DomainResource.class, domainName, now); + Optional superordinateDomain = + loadByForeignKey(DomainResource.class, domainName, now); // Hosts can't be linked if domains import hasn't been run checkState( - superordinateDomain != null, "Superordinate domain does not exist: %s", domainName); - return Optional.of(superordinateDomain); + superordinateDomain.isPresent(), + "Superordinate domain does not exist or is deleted: %s", + domainName); + return superordinateDomain; } } @@ -209,10 +212,4 @@ public class RdeHostLinkAction implements Runnable { SUPERORDINATE_DOMAIN_IN_PENDING_DELETE, HOST_LINKED; } - - private static class HostLinkException extends RuntimeException { - HostLinkException(String hostname, String xml, Throwable cause) { - super(String.format("Error linking host %s; xml=%s", hostname, xml), cause); - } - } } diff --git a/java/google/registry/reporting/ReportingModule.java b/java/google/registry/reporting/ReportingModule.java index b02fd7028..47890971b 100644 --- a/java/google/registry/reporting/ReportingModule.java +++ b/java/google/registry/reporting/ReportingModule.java @@ -28,20 +28,30 @@ import google.registry.request.Parameter; import google.registry.util.Clock; import java.util.Optional; import javax.servlet.http.HttpServletRequest; +import org.joda.time.DateTimeZone; +import org.joda.time.LocalDate; import org.joda.time.YearMonth; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; /** Dagger module for injecting common settings for all reporting tasks. */ @Module public class ReportingModule { public static final String BEAM_QUEUE = "beam-reporting"; + /** * The request parameter name used by reporting actions that takes a year/month parameter, which * defaults to the last month. */ + // TODO(b/120497263): remove this and replace with the date public static final String PARAM_YEAR_MONTH = "yearMonth"; + + /** + * The request parameter name used by reporting actions that take a local date as a parameter, + * which defaults to the current date. + */ + public static final String PARAM_DATE = "date"; + /** The request parameter specifying the jobId for a running Dataflow pipeline. */ public static final String PARAM_JOB_ID = "jobId"; @@ -56,10 +66,9 @@ public class ReportingModule { @Provides @Parameter(PARAM_YEAR_MONTH) static Optional provideYearMonthOptional(HttpServletRequest req) { - DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM"); Optional optionalYearMonthStr = extractOptionalParameter(req, PARAM_YEAR_MONTH); try { - return optionalYearMonthStr.map(s -> YearMonth.parse(s, formatter)); + return optionalYearMonthStr.map(s -> YearMonth.parse(s, ISODateTimeFormat.yearMonth())); } catch (IllegalArgumentException e) { throw new BadRequestException( String.format( @@ -74,18 +83,42 @@ public class ReportingModule { */ @Provides static YearMonth provideYearMonth( - @Parameter(PARAM_YEAR_MONTH) Optional yearMonthOptional, Clock clock) { - return yearMonthOptional.orElseGet(() -> new YearMonth(clock.nowUtc().minusMonths(1))); + @Parameter(PARAM_YEAR_MONTH) Optional yearMonthOptional, LocalDate date) { + return yearMonthOptional.orElseGet(() -> new YearMonth(date.minusMonths(1))); + } + + /** Extracts an optional date in yyyy-MM-dd format from the request. */ + @Provides + @Parameter(PARAM_DATE) + static Optional provideDateOptional(HttpServletRequest req) { + Optional optionalDateString = extractOptionalParameter(req, PARAM_DATE); + try { + return optionalDateString.map(s -> LocalDate.parse(s, ISODateTimeFormat.yearMonthDay())); + } catch (IllegalArgumentException e) { + throw new BadRequestException( + String.format( + "date must be in yyyy-MM-dd format, got %s instead", + optionalDateString.orElse("UNSPECIFIED LOCAL DATE")), + e); + } + } + + /** + * Provides the local date in yyyy-MM-dd format, if not specified in the request, defaults to the + * current date. + */ + @Provides + static LocalDate provideDate( + @Parameter(PARAM_DATE) Optional dateOptional, Clock clock) { + return dateOptional.orElseGet(() -> new LocalDate(clock.nowUtc(), DateTimeZone.UTC)); } /** Constructs a {@link Dataflow} API client with default settings. */ @Provides static Dataflow provideDataflow( @DefaultCredential GoogleCredential credential, @Config("projectId") String projectId) { - return new Dataflow.Builder(credential.getTransport(), credential.getJsonFactory(), credential) .setApplicationName(String.format("%s billing", projectId)) .build(); } - } diff --git a/java/google/registry/reporting/ReportingUtils.java b/java/google/registry/reporting/ReportingUtils.java index 279a3695b..233e18017 100644 --- a/java/google/registry/reporting/ReportingUtils.java +++ b/java/google/registry/reporting/ReportingUtils.java @@ -16,6 +16,7 @@ package google.registry.reporting; import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.TaskOptions; +import java.util.Map; import org.joda.time.Duration; import org.joda.time.YearMonth; @@ -25,15 +26,13 @@ public class ReportingUtils { private static final int ENQUEUE_DELAY_MINUTES = 10; /** Enqueues a task that takes a Beam jobId and the {@link YearMonth} as parameters. */ - public static void enqueueBeamReportingTask(String path, String jobId, YearMonth yearMonth) { + public static void enqueueBeamReportingTask(String path, Map parameters) { TaskOptions publishTask = TaskOptions.Builder.withUrl(path) .method(TaskOptions.Method.POST) // Dataflow jobs tend to take about 10 minutes to complete. - .countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis()) - .param(ReportingModule.PARAM_JOB_ID, jobId) - // Need to pass this through to ensure transitive yearMonth dependencies are satisfied. - .param(ReportingModule.PARAM_YEAR_MONTH, yearMonth.toString()); + .countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis()); + parameters.forEach(publishTask::param); QueueFactory.getQueue(ReportingModule.BEAM_QUEUE).add(publishTask); } } diff --git a/java/google/registry/reporting/billing/BillingEmailUtils.java b/java/google/registry/reporting/billing/BillingEmailUtils.java index ae91c7981..e15c26a72 100644 --- a/java/google/registry/reporting/billing/BillingEmailUtils.java +++ b/java/google/registry/reporting/billing/BillingEmailUtils.java @@ -44,10 +44,11 @@ class BillingEmailUtils { private final SendEmailService emailService; private final YearMonth yearMonth; - private final String alertSenderAddress; + private final String outgoingEmailAddress; private final String alertRecipientAddress; private final ImmutableList invoiceEmailRecipients; private final String billingBucket; + private final String invoiceFilePrefix; private final String invoiceDirectoryPrefix; private final GcsUtils gcsUtils; private final Retrier retrier; @@ -56,19 +57,21 @@ class BillingEmailUtils { BillingEmailUtils( SendEmailService emailService, YearMonth yearMonth, - @Config("alertSenderEmailAddress") String alertSenderAddress, + @Config("gSuiteOutgoingEmailAddress") String outgoingEmailAddress, @Config("alertRecipientEmailAddress") String alertRecipientAddress, @Config("invoiceEmailRecipients") ImmutableList invoiceEmailRecipients, @Config("billingBucket") String billingBucket, + @Config("invoiceFilePrefix") String invoiceFilePrefix, @InvoiceDirectoryPrefix String invoiceDirectoryPrefix, GcsUtils gcsUtils, Retrier retrier) { this.emailService = emailService; this.yearMonth = yearMonth; - this.alertSenderAddress = alertSenderAddress; + this.outgoingEmailAddress = outgoingEmailAddress; this.alertRecipientAddress = alertRecipientAddress; this.invoiceEmailRecipients = invoiceEmailRecipients; this.billingBucket = billingBucket; + this.invoiceFilePrefix = invoiceFilePrefix; this.invoiceDirectoryPrefix = invoiceDirectoryPrefix; this.gcsUtils = gcsUtils; this.retrier = retrier; @@ -80,23 +83,22 @@ class BillingEmailUtils { retrier.callWithRetry( () -> { String invoiceFile = - String.format( - "%s-%s.csv", BillingModule.OVERALL_INVOICE_PREFIX, yearMonth.toString()); + String.format("%s-%s.csv", invoiceFilePrefix, yearMonth); GcsFilename invoiceFilename = new GcsFilename(billingBucket, invoiceDirectoryPrefix + invoiceFile); try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) { Message msg = emailService.createMessage(); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); for (String recipient : invoiceEmailRecipients) { msg.addRecipient(RecipientType.TO, new InternetAddress(recipient)); } msg.setSubject( - String.format("Domain Registry invoice data %s", yearMonth.toString())); + String.format("Domain Registry invoice data %s", yearMonth)); Multipart multipart = new MimeMultipart(); BodyPart textPart = new MimeBodyPart(); textPart.setText( String.format( - "Attached is the %s invoice for the domain registry.", yearMonth.toString())); + "Attached is the %s invoice for the domain registry.", yearMonth)); multipart.addBodyPart(textPart); BodyPart invoicePart = new MimeBodyPart(); String invoiceData = CharStreams.toString(new InputStreamReader(in, UTF_8)); @@ -126,9 +128,9 @@ class BillingEmailUtils { retrier.callWithRetry( () -> { Message msg = emailService.createMessage(); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); msg.addRecipient(RecipientType.TO, new InternetAddress(alertRecipientAddress)); - msg.setSubject(String.format("Billing Pipeline Alert: %s", yearMonth.toString())); + msg.setSubject(String.format("Billing Pipeline Alert: %s", yearMonth)); msg.setText(body); emailService.sendMessage(msg); return null; diff --git a/java/google/registry/reporting/billing/BillingModule.java b/java/google/registry/reporting/billing/BillingModule.java index b57c75f09..b8fa95d0f 100644 --- a/java/google/registry/reporting/billing/BillingModule.java +++ b/java/google/registry/reporting/billing/BillingModule.java @@ -32,7 +32,6 @@ import org.joda.time.YearMonth; public final class BillingModule { public static final String DETAIL_REPORT_PREFIX = "invoice_details"; - public static final String OVERALL_INVOICE_PREFIX = "CRR-INV"; public static final String INVOICES_DIRECTORY = "invoices"; static final String PARAM_SHOULD_PUBLISH = "shouldPublish"; diff --git a/java/google/registry/reporting/billing/GenerateInvoicesAction.java b/java/google/registry/reporting/billing/GenerateInvoicesAction.java index ed325a6dc..8d573eaeb 100644 --- a/java/google/registry/reporting/billing/GenerateInvoicesAction.java +++ b/java/google/registry/reporting/billing/GenerateInvoicesAction.java @@ -28,11 +28,13 @@ import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; +import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; +import java.util.Map; import javax.inject.Inject; import org.joda.time.YearMonth; @@ -105,7 +107,13 @@ public class GenerateInvoicesAction implements Runnable { logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString()); String jobId = launchResponse.getJob().getId(); if (shouldPublish) { - enqueueBeamReportingTask(PublishInvoicesAction.PATH, jobId, yearMonth); + Map beamTaskParameters = + ImmutableMap.of( + ReportingModule.PARAM_JOB_ID, + jobId, + ReportingModule.PARAM_YEAR_MONTH, + yearMonth.toString()); + enqueueBeamReportingTask(PublishInvoicesAction.PATH, beamTaskParameters); } } catch (IOException e) { logger.atWarning().withCause(e).log("Template Launch failed"); diff --git a/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java b/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java index 18a1aeaf2..9aeafe905 100644 --- a/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java +++ b/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java @@ -29,9 +29,7 @@ import org.joda.time.YearMonth; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; -/** - * Utility class that produces SQL queries used to generate activity reports from Bigquery. - */ +/** Utility class that produces SQL queries used to generate activity reports from Bigquery. */ public final class ActivityReportingQueryBuilder implements QueryBuilder { // Names for intermediary tables for overall activity reporting query. @@ -42,20 +40,23 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder { static final String WHOIS_COUNTS = "whois_counts"; static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation"; - @Inject @Config("projectId") String projectId; + @Inject + @Config("projectId") + String projectId; @Inject YearMonth yearMonth; - @Inject ActivityReportingQueryBuilder() {} + @Inject DnsCountQueryCoordinator dnsCountQueryCoordinator; + + @Inject + ActivityReportingQueryBuilder() {} /** Returns the aggregate query which generates the activity report from the saved view. */ @Override public String getReportQuery() { return String.format( "#standardSQL\nSELECT * FROM `%s.%s.%s`", - projectId, - ICANN_REPORTING_DATA_SET, - getTableName(ACTIVITY_REPORT_AGGREGATION)); + projectId, ICANN_REPORTING_DATA_SET, getTableName(ACTIVITY_REPORT_AGGREGATION)); } /** Sets the month we're doing activity reporting for, and returns the view query map. */ @@ -67,6 +68,10 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder { return createQueryMap(firstDayOfMonth, lastDayOfMonth); } + public void prepareForQuery() throws Exception { + dnsCountQueryCoordinator.prepareForQuery(); + } + /** Returns a map from view name to its associated SQL query. */ private ImmutableMap createQueryMap( LocalDate firstDayOfMonth, LocalDate lastDayOfMonth) { @@ -80,8 +85,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder { .build(); queriesBuilder.put(getTableName(REGISTRAR_OPERATING_STATUS), operationalRegistrarsQuery); - String dnsCountsQuery = - SqlTemplate.create(getQueryFromFile("dns_counts.sql")).build(); + String dnsCountsQuery = dnsCountQueryCoordinator.createQuery(); queriesBuilder.put(getTableName(DNS_COUNTS), dnsCountsQuery); // Convert reportingMonth into YYYYMMDD format for Bigquery table partition pattern-matching. @@ -133,7 +137,6 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder { return queriesBuilder.build(); } - /** Returns the table name of the query, suffixed with the yearMonth in _yyyyMM format. */ private String getTableName(String queryName) { return String.format("%s_%s", queryName, DateTimeFormat.forPattern("yyyyMM").print(yearMonth)); diff --git a/java/google/registry/reporting/icann/BasicDnsCountQueryCoordinator.java b/java/google/registry/reporting/icann/BasicDnsCountQueryCoordinator.java new file mode 100644 index 000000000..b35150b3f --- /dev/null +++ b/java/google/registry/reporting/icann/BasicDnsCountQueryCoordinator.java @@ -0,0 +1,38 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.icann; + +import com.google.common.io.Resources; +import google.registry.util.ResourceUtils; +import google.registry.util.SqlTemplate; + +/** + * DNS Count query for the basic case. + */ +public class BasicDnsCountQueryCoordinator implements DnsCountQueryCoordinator { + + BasicDnsCountQueryCoordinator(DnsCountQueryCoordinator.Params params) {} + + @Override + public String createQuery() { + return SqlTemplate.create( + ResourceUtils.readResourceUtf8( + Resources.getResource(this.getClass(), "sql/" + "dns_counts.sql"))) + .build(); + } + + @Override + public void prepareForQuery() throws Exception {} +} diff --git a/java/google/registry/reporting/icann/DnsCountQueryCoordinator.java b/java/google/registry/reporting/icann/DnsCountQueryCoordinator.java new file mode 100644 index 000000000..1f2f9c10a --- /dev/null +++ b/java/google/registry/reporting/icann/DnsCountQueryCoordinator.java @@ -0,0 +1,58 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.icann; + +import google.registry.bigquery.BigqueryConnection; +import org.joda.time.YearMonth; + +/** + * Methods for preparing and querying DNS statistics. + * + *

DNS systems may have different ways of providing this information, so it's useful to + * modularize this. + * + *

Derived classes must provide a constructor that accepts a + * {@link google.registry.reporting.icann.DnsCountQueryCoordinator.Params}. To override this, + * define dnsCountQueryCoordinatorClass in your config file. + */ +public interface DnsCountQueryCoordinator { + + /** + * Class to carry parameters for a new coordinator. + * + * If your report query requires any additional parameters, add them here. + */ + public class Params { + public BigqueryConnection bigquery; + + /** The year and month of the report. */ + public YearMonth yearMonth; + + /** The Google Cloud project id. */ + public String projectId; + + public Params(BigqueryConnection bigquery, YearMonth yearMonth, String projectId) { + this.bigquery = bigquery; + this.yearMonth = yearMonth; + this.projectId = projectId; + } + } + + /** Creates the string used to query bigtable for DNS count information. */ + String createQuery(); + + /** Do any necessry preparation for the DNS query. */ + void prepareForQuery() throws Exception; +} diff --git a/java/google/registry/reporting/icann/DnsCountQueryCoordinatorModule.java b/java/google/registry/reporting/icann/DnsCountQueryCoordinatorModule.java new file mode 100644 index 000000000..bbf7241ff --- /dev/null +++ b/java/google/registry/reporting/icann/DnsCountQueryCoordinatorModule.java @@ -0,0 +1,42 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.icann; + +import static google.registry.util.TypeUtils.getClassFromString; +import static google.registry.util.TypeUtils.instantiate; + +import dagger.Module; +import dagger.Provides; +import google.registry.bigquery.BigqueryConnection; +import google.registry.config.RegistryConfig.Config; +import org.joda.time.YearMonth; + +/** Dagger module to provide the DnsCountQueryCoordinator. */ +@Module +public class DnsCountQueryCoordinatorModule { + + @Provides + static DnsCountQueryCoordinator provideDnsCountQueryCoordinator( + @Config("dnsCountQueryCoordinatorClass") String customClass, + BigqueryConnection bigquery, + YearMonth yearMonth, + @Config("projectId") String projectId) { + DnsCountQueryCoordinator.Params params = + new DnsCountQueryCoordinator.Params(bigquery, yearMonth, projectId); + DnsCountQueryCoordinator result = + instantiate(getClassFromString(customClass, DnsCountQueryCoordinator.class), params); + return result; + } +} diff --git a/java/google/registry/reporting/icann/IcannReportingStager.java b/java/google/registry/reporting/icann/IcannReportingStager.java index a988c29c6..1f998ad61 100644 --- a/java/google/registry/reporting/icann/IcannReportingStager.java +++ b/java/google/registry/reporting/icann/IcannReportingStager.java @@ -83,6 +83,10 @@ public class IcannReportingStager { QueryBuilder queryBuilder = (reportType == ReportType.ACTIVITY) ? activityQueryBuilder : transactionsQueryBuilder; + if (reportType == ReportType.ACTIVITY) { + // Prepare for the DNS count query, which may have special needs. + activityQueryBuilder.prepareForQuery(); + } ImmutableMap viewQueryMap = queryBuilder.getViewQueryMap(); // Generate intermediary views diff --git a/java/google/registry/reporting/icann/ReportingEmailUtils.java b/java/google/registry/reporting/icann/ReportingEmailUtils.java index 9b248f94d..c011b8bfa 100644 --- a/java/google/registry/reporting/icann/ReportingEmailUtils.java +++ b/java/google/registry/reporting/icann/ReportingEmailUtils.java @@ -25,7 +25,7 @@ import javax.mail.internet.InternetAddress; /** Static utils for emailing reporting results. */ public class ReportingEmailUtils { - @Inject @Config("alertSenderEmailAddress") String sender; + @Inject @Config("gSuiteOutgoingEmailAddress") String sender; @Inject @Config("alertRecipientEmailAddress") String recipient; @Inject SendEmailService emailService; @Inject ReportingEmailUtils() {} diff --git a/java/google/registry/reporting/spec11/BUILD b/java/google/registry/reporting/spec11/BUILD index 6317887a2..a7b4f296a 100644 --- a/java/google/registry/reporting/spec11/BUILD +++ b/java/google/registry/reporting/spec11/BUILD @@ -20,6 +20,7 @@ java_library( "@com_google_apis_google_api_services_dataflow", "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_tools_appengine_gcs_client", + "@com_google_auto_value", "@com_google_dagger", "@com_google_flogger", "@com_google_flogger_system_backend", diff --git a/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java b/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java index 2a41c3b67..57d701519 100644 --- a/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java +++ b/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java @@ -28,12 +28,14 @@ import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; +import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; +import java.util.Map; import javax.inject.Inject; -import org.joda.time.YearMonth; +import org.joda.time.LocalDate; /** * Invokes the {@code Spec11Pipeline} Beam template via the REST api. @@ -53,7 +55,7 @@ public class GenerateSpec11ReportAction implements Runnable { private final String spec11TemplateUrl; private final String jobZone; private final String apiKey; - private final YearMonth yearMonth; + private final LocalDate date; private final Response response; private final Dataflow dataflow; @@ -64,7 +66,7 @@ public class GenerateSpec11ReportAction implements Runnable { @Config("spec11TemplateUrl") String spec11TemplateUrl, @Config("defaultJobZone") String jobZone, @Key("safeBrowsingAPIKey") String apiKey, - YearMonth yearMonth, + LocalDate date, Response response, Dataflow dataflow) { this.projectId = projectId; @@ -72,7 +74,7 @@ public class GenerateSpec11ReportAction implements Runnable { this.spec11TemplateUrl = spec11TemplateUrl; this.jobZone = jobZone; this.apiKey = apiKey; - this.yearMonth = yearMonth; + this.date = date; this.response = response; this.dataflow = dataflow; } @@ -82,14 +84,14 @@ public class GenerateSpec11ReportAction implements Runnable { try { LaunchTemplateParameters params = new LaunchTemplateParameters() - .setJobName(String.format("spec11_%s", yearMonth.toString())) + .setJobName(String.format("spec11_%s", date)) .setEnvironment( new RuntimeEnvironment() .setZone(jobZone) .setTempLocation(beamBucketUrl + "/temporary")) .setParameters( ImmutableMap.of( - "safeBrowsingApiKey", apiKey, "yearMonth", yearMonth.toString("yyyy-MM"))); + "safeBrowsingApiKey", apiKey, ReportingModule.PARAM_DATE, date.toString())); LaunchTemplateResponse launchResponse = dataflow .projects() @@ -97,8 +99,13 @@ public class GenerateSpec11ReportAction implements Runnable { .launch(projectId, params) .setGcsPath(spec11TemplateUrl) .execute(); - enqueueBeamReportingTask( - PublishSpec11ReportAction.PATH, launchResponse.getJob().getId(), yearMonth); + Map beamTaskParameters = + ImmutableMap.of( + ReportingModule.PARAM_JOB_ID, + launchResponse.getJob().getId(), + ReportingModule.PARAM_DATE, + date.toString()); + enqueueBeamReportingTask(PublishSpec11ReportAction.PATH, beamTaskParameters); logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString()); } catch (IOException e) { logger.atWarning().withCause(e).log("Template Launch failed"); diff --git a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java index d8866617f..7a7870062 100644 --- a/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java +++ b/java/google/registry/reporting/spec11/PublishSpec11ReportAction.java @@ -22,6 +22,7 @@ import static javax.servlet.http.HttpServletResponse.SC_OK; import com.google.api.services.dataflow.Dataflow; import com.google.api.services.dataflow.model.Job; +import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; @@ -31,15 +32,17 @@ import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import java.io.IOException; +import java.util.List; import javax.inject.Inject; -import org.joda.time.YearMonth; +import org.joda.time.LocalDate; +import org.json.JSONException; /** * Retries until a {@code Dataflow} job with a given {@code jobId} completes, continuing the Spec11 * pipeline accordingly. * - *

This calls {@link Spec11EmailUtils#emailSpec11Reports()} on success or {@link - * Spec11EmailUtils#sendAlertEmail(String, String)} on failure. + *

This calls {@link Spec11EmailUtils#emailSpec11Reports(String, String, List)} ()} on success or + * {@link Spec11EmailUtils#sendAlertEmail(String, String)} on failure. */ @Action(path = PublishSpec11ReportAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_OR_ADMIN) public class PublishSpec11ReportAction implements Runnable { @@ -51,26 +54,32 @@ public class PublishSpec11ReportAction implements Runnable { private static final String JOB_FAILED = "JOB_STATE_FAILED"; private final String projectId; + private final String spec11EmailBodyTemplate; private final String jobId; private final Spec11EmailUtils emailUtils; + private final Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser; private final Dataflow dataflow; private final Response response; - private final YearMonth yearMonth; + private final LocalDate date; @Inject PublishSpec11ReportAction( @Config("projectId") String projectId, + @Config("spec11EmailBodyTemplate") String spec11EmailBodyTemplate, @Parameter(ReportingModule.PARAM_JOB_ID) String jobId, Spec11EmailUtils emailUtils, + Spec11RegistrarThreatMatchesParser spec11RegistrarThreatMatchesParser, Dataflow dataflow, Response response, - YearMonth yearMonth) { + LocalDate date) { this.projectId = projectId; + this.spec11EmailBodyTemplate = spec11EmailBodyTemplate; this.jobId = jobId; this.emailUtils = emailUtils; + this.spec11RegistrarThreatMatchesParser = spec11RegistrarThreatMatchesParser; this.dataflow = dataflow; this.response = response; - this.yearMonth = yearMonth; + this.date = date; } @Override @@ -81,32 +90,41 @@ public class PublishSpec11ReportAction implements Runnable { String state = job.getCurrentState(); switch (state) { case JOB_DONE: - logger.atInfo().log("Dataflow job %s finished successfully, publishing results.", jobId); + logger.atInfo().log( + "Dataflow job %s finished successfully, publishing results if appropriate.", jobId); response.setStatus(SC_OK); - emailUtils.emailSpec11Reports(); + if (shouldSendSpec11Email()) { + ImmutableList matchesList = + spec11RegistrarThreatMatchesParser.getRegistrarThreatMatches(); + String subject = String.format("Google Registry Monthly Threat Detector [%s]", date); + emailUtils.emailSpec11Reports(spec11EmailBodyTemplate, subject, matchesList); + } break; case JOB_FAILED: logger.atSevere().log("Dataflow job %s finished unsuccessfully.", jobId); response.setStatus(SC_NO_CONTENT); emailUtils.sendAlertEmail( - String.format("Spec11 Dataflow Pipeline Failure %s", yearMonth.toString()), - String.format( - "Spec11 %s job %s ended in status failure.", yearMonth.toString(), jobId)); + String.format("Spec11 Dataflow Pipeline Failure %s", date), + String.format("Spec11 %s job %s ended in status failure.", date, jobId)); break; default: logger.atInfo().log("Job in non-terminal state %s, retrying:", state); response.setStatus(SC_NOT_MODIFIED); break; } - } catch (IOException e) { + } catch (IOException | JSONException e) { logger.atSevere().withCause(e).log("Failed to publish Spec11 reports."); emailUtils.sendAlertEmail( - String.format("Spec11 Publish Failure %s", yearMonth.toString()), - String.format( - "Spec11 %s publish action failed due to %s", yearMonth.toString(), e.getMessage())); + String.format("Spec11 Publish Failure %s", date), + String.format("Spec11 %s publish action failed due to %s", date, e.getMessage())); response.setStatus(SC_INTERNAL_SERVER_ERROR); response.setContentType(MediaType.PLAIN_TEXT_UTF_8); response.setPayload(String.format("Template launch failed: %s", e.getMessage())); } } + + private boolean shouldSendSpec11Email() { + // TODO(b/120496893): send emails every day with the diff content + return date.getDayOfMonth() == 2; + } } diff --git a/java/google/registry/reporting/spec11/RegistrarThreatMatches.java b/java/google/registry/reporting/spec11/RegistrarThreatMatches.java new file mode 100644 index 000000000..c1a2f68fa --- /dev/null +++ b/java/google/registry/reporting/spec11/RegistrarThreatMatches.java @@ -0,0 +1,33 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import com.google.auto.value.AutoValue; +import google.registry.beam.spec11.ThreatMatch; +import java.util.List; + +/** Value class representing the registrar and list-of-threat-matches pair stored in GCS. */ +@AutoValue +public abstract class RegistrarThreatMatches { + + public abstract String registrarEmailAddress(); + + public abstract List threatMatches(); + + static RegistrarThreatMatches create( + String registrarEmailAddress, List threatMatches) { + return new AutoValue_RegistrarThreatMatches(registrarEmailAddress, threatMatches); + } +} diff --git a/java/google/registry/reporting/spec11/Spec11EmailUtils.java b/java/google/registry/reporting/spec11/Spec11EmailUtils.java index 76b716446..7b54068e3 100644 --- a/java/google/registry/reporting/spec11/Spec11EmailUtils.java +++ b/java/google/registry/reporting/spec11/Spec11EmailUtils.java @@ -15,88 +15,59 @@ package google.registry.reporting.spec11; import static com.google.common.base.Throwables.getRootCause; -import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.common.collect.ImmutableList; -import com.google.common.io.CharStreams; -import google.registry.beam.spec11.Spec11Pipeline; import google.registry.beam.spec11.ThreatMatch; import google.registry.config.RegistryConfig.Config; -import google.registry.gcs.GcsUtils; -import google.registry.reporting.spec11.Spec11Module.Spec11ReportDirectory; import google.registry.util.Retrier; import google.registry.util.SendEmailService; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.util.List; import javax.inject.Inject; import javax.mail.Message; import javax.mail.Message.RecipientType; import javax.mail.MessagingException; import javax.mail.internet.InternetAddress; -import org.joda.time.YearMonth; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import org.joda.time.LocalDate; /** Provides e-mail functionality for Spec11 tasks, such as sending Spec11 reports to registrars. */ public class Spec11EmailUtils { private final SendEmailService emailService; - private final YearMonth yearMonth; - private final String alertSenderAddress; + private final LocalDate date; + private final String outgoingEmailAddress; private final String alertRecipientAddress; private final String spec11ReplyToAddress; - private final String reportingBucket; - private final String spec11ReportDirectory; - private final String spec11EmailBodyTemplate; - private final GcsUtils gcsUtils; private final Retrier retrier; @Inject Spec11EmailUtils( SendEmailService emailService, - YearMonth yearMonth, - @Config("alertSenderEmailAddress") String alertSenderAddress, + LocalDate date, + @Config("gSuiteOutgoingEmailAddress") String outgoingEmailAddress, @Config("alertRecipientEmailAddress") String alertRecipientAddress, @Config("spec11ReplyToEmailAddress") String spec11ReplyToAddress, - @Config("spec11EmailBodyTemplate") String spec11EmailBodyTemplate, - @Config("reportingBucket") String reportingBucket, - @Spec11ReportDirectory String spec11ReportDirectory, - GcsUtils gcsUtils, Retrier retrier) { this.emailService = emailService; - this.yearMonth = yearMonth; - this.alertSenderAddress = alertSenderAddress; + this.date = date; + this.outgoingEmailAddress = outgoingEmailAddress; this.alertRecipientAddress = alertRecipientAddress; this.spec11ReplyToAddress = spec11ReplyToAddress; - this.reportingBucket = reportingBucket; - this.spec11ReportDirectory = spec11ReportDirectory; - this.spec11EmailBodyTemplate = spec11EmailBodyTemplate; - this.gcsUtils = gcsUtils; this.retrier = retrier; } /** - * Processes a Spec11 report on GCS for a given month and e-mails registrars based on the - * contents. + * Processes a list of registrar/list-of-threat pairings and sends a notification email to the + * appropriate address. */ - void emailSpec11Reports() { + void emailSpec11Reports( + String spec11EmailBodyTemplate, + String subject, + List registrarThreatMatchesList) { try { retrier.callWithRetry( () -> { - // Grab the file as an inputstream - GcsFilename spec11ReportFilename = - new GcsFilename(reportingBucket, spec11ReportDirectory); - try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) { - ImmutableList reportLines = - ImmutableList.copyOf( - CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n")); - // Iterate from 1 to size() to skip the header at line 0. - for (int i = 1; i < reportLines.size(); i++) { - emailRegistrar(reportLines.get(i)); - } + for (RegistrarThreatMatches registrarThreatMatches : registrarThreatMatchesList) { + emailRegistrar(spec11EmailBodyTemplate, subject, registrarThreatMatches); } }, IOException.class, @@ -104,25 +75,21 @@ public class Spec11EmailUtils { } catch (Throwable e) { // Send an alert with the root cause, unwrapping the retrier's RuntimeException sendAlertEmail( - String.format("Spec11 Emailing Failure %s", yearMonth.toString()), - String.format( - "Emailing spec11 reports failed due to %s", - getRootCause(e).getMessage())); + String.format("Spec11 Emailing Failure %s", date), + String.format("Emailing spec11 reports failed due to %s", getRootCause(e).getMessage())); throw new RuntimeException("Emailing spec11 report failed", e); } sendAlertEmail( - String.format("Spec11 Pipeline Success %s", yearMonth.toString()), + String.format("Spec11 Pipeline Success %s", date), "Spec11 reporting completed successfully."); } - private void emailRegistrar(String line) throws MessagingException, JSONException { - // Parse the Spec11 report JSON - JSONObject reportJSON = new JSONObject(line); - String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD); - JSONArray threatMatches = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD); + private void emailRegistrar( + String spec11EmailBodyTemplate, String subject, RegistrarThreatMatches registrarThreatMatches) + throws MessagingException { + String registrarEmail = registrarThreatMatches.registrarEmailAddress(); StringBuilder threatList = new StringBuilder(); - for (int i = 0; i < threatMatches.length(); i++) { - ThreatMatch threatMatch = ThreatMatch.fromJSON(threatMatches.getJSONObject(i)); + for (ThreatMatch threatMatch : registrarThreatMatches.threatMatches()) { threatList.append( String.format( "%s - %s\n", threatMatch.fullyQualifiedDomainName(), threatMatch.threatType())); @@ -132,10 +99,9 @@ public class Spec11EmailUtils { .replace("{REPLY_TO_EMAIL}", spec11ReplyToAddress) .replace("{LIST_OF_THREATS}", threatList.toString()); Message msg = emailService.createMessage(); - msg.setSubject( - String.format("Google Registry Monthly Threat Detector [%s]", yearMonth.toString())); - msg.setText(body.toString()); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setSubject(subject); + msg.setText(body); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); msg.addRecipient(RecipientType.TO, new InternetAddress(registrarEmail)); msg.addRecipient(RecipientType.BCC, new InternetAddress(spec11ReplyToAddress)); emailService.sendMessage(msg); @@ -147,7 +113,7 @@ public class Spec11EmailUtils { retrier.callWithRetry( () -> { Message msg = emailService.createMessage(); - msg.setFrom(new InternetAddress(alertSenderAddress)); + msg.setFrom(new InternetAddress(outgoingEmailAddress)); msg.addRecipient(RecipientType.TO, new InternetAddress(alertRecipientAddress)); msg.setSubject(subject); msg.setText(body); diff --git a/java/google/registry/reporting/spec11/Spec11Module.java b/java/google/registry/reporting/spec11/Spec11Module.java index 0135459e6..4fee57004 100644 --- a/java/google/registry/reporting/spec11/Spec11Module.java +++ b/java/google/registry/reporting/spec11/Spec11Module.java @@ -22,21 +22,21 @@ import google.registry.beam.spec11.Spec11Pipeline; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import javax.inject.Qualifier; -import org.joda.time.YearMonth; +import org.joda.time.LocalDate; /** Module for dependencies required by Spec11 reporting. */ @Module public class Spec11Module { @Provides - @Spec11ReportDirectory - static String provideDirectoryPrefix(YearMonth yearMonth) { - return Spec11Pipeline.getSpec11Subdirectory(yearMonth.toString("yyyy-MM")); + @Spec11ReportFilePath + static String provideSpec11ReportFilePath(LocalDate localDate) { + return Spec11Pipeline.getSpec11ReportFilePath(localDate); } /** Dagger qualifier for the subdirectory we stage to/upload from for Spec11 reports. */ @Qualifier @Documented @Retention(RUNTIME) - @interface Spec11ReportDirectory {} + @interface Spec11ReportFilePath {} } diff --git a/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java new file mode 100644 index 000000000..4e16ebc01 --- /dev/null +++ b/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java @@ -0,0 +1,78 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.reporting.spec11; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.common.collect.ImmutableList; +import com.google.common.io.CharStreams; +import google.registry.beam.spec11.Spec11Pipeline; +import google.registry.beam.spec11.ThreatMatch; +import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import javax.inject.Inject; +import org.joda.time.LocalDate; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/** Parser to retrieve which registrar-threat matches we should notify via email */ +public class Spec11RegistrarThreatMatchesParser { + + private final LocalDate date; + private final GcsUtils gcsUtils; + private final String reportingBucket; + + @Inject + public Spec11RegistrarThreatMatchesParser( + LocalDate date, GcsUtils gcsUtils, @Config("reportingBucket") String reportingBucket) { + this.date = date; + this.gcsUtils = gcsUtils; + this.reportingBucket = reportingBucket; + } + + /** Gets the list of registrar:set-of-threat-match pairings from the file in GCS. */ + public ImmutableList getRegistrarThreatMatches() + throws IOException, JSONException { + // TODO(b/120078223): this should only be the diff of this run and the prior run. + GcsFilename spec11ReportFilename = + new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(date)); + ImmutableList.Builder builder = ImmutableList.builder(); + try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) { + ImmutableList reportLines = + ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n")); + // Iterate from 1 to size() to skip the header at line 0. + for (int i = 1; i < reportLines.size(); i++) { + builder.add(parseRegistrarThreatMatch(reportLines.get(i))); + } + return builder.build(); + } + } + + private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException { + JSONObject reportJSON = new JSONObject(line); + String registrarEmail = reportJSON.getString(Spec11Pipeline.REGISTRAR_EMAIL_FIELD); + JSONArray threatMatchesArray = reportJSON.getJSONArray(Spec11Pipeline.THREAT_MATCHES_FIELD); + ImmutableList.Builder threatMatches = ImmutableList.builder(); + for (int i = 0; i < threatMatchesArray.length(); i++) { + threatMatches.add(ThreatMatch.fromJSON(threatMatchesArray.getJSONObject(i))); + } + return RegistrarThreatMatches.create(registrarEmail, threatMatches.build()); + } +} diff --git a/java/google/registry/repositories.bzl b/java/google/registry/repositories.bzl index d05fb830a..cd5a23c68 100644 --- a/java/google/registry/repositories.bzl +++ b/java/google/registry/repositories.bzl @@ -15,6 +15,7 @@ """External dependencies for Nomulus.""" +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external") def domain_registry_bazel_check(): @@ -31,6 +32,7 @@ def domain_registry_repositories( omit_com_google_api_client_jackson2 = False, omit_com_google_api_client_java6 = False, omit_com_google_api_client_servlet = False, + omit_com_google_apis_google_api_services_appengine = False, omit_com_google_apis_google_api_services_admin_directory = False, omit_com_google_apis_google_api_services_bigquery = False, omit_com_google_apis_google_api_services_clouddebugger = False, @@ -58,6 +60,7 @@ def domain_registry_repositories( omit_com_google_auto_factory = False, omit_com_google_auto_service = False, omit_com_google_auto_value = False, + omit_com_google_code_gson = False, omit_com_google_cloud_bigdataoss_gcsio = False, omit_com_google_cloud_bigdataoss_util = False, omit_com_google_code_findbugs_jsr305 = False, @@ -84,6 +87,7 @@ def domain_registry_repositories( omit_com_google_oauth_client_java6 = False, omit_com_google_oauth_client_jetty = False, omit_com_google_oauth_client_servlet = False, + omit_com_google_protobuf = False, omit_com_google_protobuf_java = False, omit_com_google_re2j = False, omit_com_google_template_soy = False, @@ -102,6 +106,7 @@ def domain_registry_repositories( omit_com_sun_xml_bind_jaxb_xjc = False, omit_com_thoughtworks_paranamer = False, omit_commons_codec = False, + omit_commons_io = False, omit_commons_logging = False, omit_dnsjava = False, omit_io_netty_buffer = False, @@ -187,6 +192,8 @@ def domain_registry_repositories( com_google_api_client_servlet() if not omit_com_google_apis_google_api_services_admin_directory: com_google_apis_google_api_services_admin_directory() + if not omit_com_google_apis_google_api_services_appengine: + com_google_apis_google_api_services_appengine() if not omit_com_google_apis_google_api_services_bigquery: com_google_apis_google_api_services_bigquery() if not omit_com_google_apis_google_api_services_clouddebugger: @@ -239,6 +246,8 @@ def domain_registry_repositories( com_google_auto_service() if not omit_com_google_auto_value: com_google_auto_value() + if not omit_com_google_code_gson: + com_google_code_gson() if not omit_com_google_cloud_bigdataoss_gcsio: com_google_cloud_bigdataoss_gcsio() if not omit_com_google_cloud_bigdataoss_util: @@ -291,6 +300,8 @@ def domain_registry_repositories( com_google_oauth_client_jetty() if not omit_com_google_oauth_client_servlet: com_google_oauth_client_servlet() + if not omit_com_google_protobuf: + com_google_protobuf() if not omit_com_google_protobuf_java: com_google_protobuf_java() if not omit_com_google_re2j: @@ -327,6 +338,8 @@ def domain_registry_repositories( com_thoughtworks_paranamer() if not omit_commons_codec: commons_codec() + if not omit_commons_io: + commons_io() if not omit_commons_logging: commons_logging() if not omit_dnsjava: @@ -466,12 +479,12 @@ def com_beust_jcommander(): def com_fasterxml_jackson_core(): java_import_external( name = "com_fasterxml_jackson_core", - jar_sha256 = "85b48d80d0ff36eecdc61ab57fe211a266b9fc326d5e172764d150e29fc99e21", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/fasterxml/jackson/core/jackson-core/2.8.5/jackson-core-2.8.5.jar", - "http://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.8.5/jackson-core-2.8.5.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "fab8746aedd6427788ee390ea04d438ec141bff7eb3476f8bdd5d9110fb2718a", + jar_urls = [ + "http://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.9.6/jackson-core-2.9.6.jar", + "http://maven.ibiblio.org/maven2/com/fasterxml/jackson/core/jackson-core/2.9.6/jackson-core-2.9.6.jar", + ], ) def com_fasterxml_jackson_core_jackson_annotations(): @@ -503,12 +516,12 @@ def com_fasterxml_jackson_core_jackson_databind(): def com_google_api_client(): java_import_external( name = "com_google_api_client", - jar_sha256 = "47c625c83a8cf97b8bbdff2acde923ff8fd3174e62aabcfc5d1b86692594ffba", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/api-client/google-api-client/1.22.0/google-api-client-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/api-client/google-api-client/1.22.0/google-api-client-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "fd1f06bc8cea64cd6e85e7a29dd632ba05c4e4ec2daae9a7115b6dbc9004fcd9", + jar_urls = [ + "http://maven.ibiblio.org/maven2/com/google/api-client/google-api-client/1.27.0/google-api-client-1.27.0.jar", + "http://repo1.maven.org/maven2/com/google/api-client/google-api-client/1.27.0/google-api-client-1.27.0.jar", + ], deps = [ "@com_google_oauth_client", "@com_google_http_client_jackson2", @@ -603,10 +616,10 @@ def com_google_api_client_java6(): java_import_external( name = "com_google_api_client_java6", licenses = ["notice"], # The Apache Software License, Version 2.0 - jar_sha256 = "df4f423f33f467d248e51deb555404771f7bc41430b2d4d1e49966c79c0b207b", + jar_sha256 = "056ef35bafebd2e2b27817be00aa08e79d24fd4ba1c7c70c2407fd2ec9582cb5", jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/api-client/google-api-client-java6/1.20.0/google-api-client-java6-1.20.0.jar", - "http://repo1.maven.org/maven2/com/google/api-client/google-api-client-java6/1.20.0/google-api-client-java6-1.20.0.jar", + "http://maven.ibiblio.org/maven2/com/google/api-client/google-api-client-java6/1.27.0/google-api-client-java6-1.27.0.jar", + "http://repo1.maven.org/maven2/com/google/api-client/google-api-client-java6/1.27.0/google-api-client-java6-1.27.0.jar", ], deps = [ "@com_google_api_client", @@ -642,6 +655,18 @@ def com_google_apis_google_api_services_admin_directory(): deps = ["@com_google_api_client"], ) +def com_google_apis_google_api_services_appengine(): + java_import_external( + name = "com_google_apis_google_api_services_appengine", + licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "15329545770163aec4f2bb0c37949a03667f06e012e2204ede22a0c2fb8f9f21", + jar_urls = [ + "http://repo1.maven.org/maven2/com/google/apis/google-api-services-appengine/v1-rev85-1.25.0/google-api-services-appengine-v1-rev85-1.25.0.jar", + "http://maven.ibiblio.org/maven2/com/google/apis/google-api-services-appengine/v1-rev85-1.25.0/google-api-services-appengine-v1-rev85-1.25.0.jar", + ], + deps = ["@com_google_api_client"], + ) + def com_google_apis_google_api_services_bigquery(): java_import_external( name = "com_google_apis_google_api_services_bigquery", @@ -1100,6 +1125,17 @@ def com_google_auto_value(): ]), ) +def com_google_code_gson(): + java_import_external( + name = "com_google_code_gson", + licenses = ["notice"], # Apache 2.0 + jar_sha256 = "233a0149fc365c9f6edbd683cfe266b19bdc773be98eabdaf6b3c924b48e7d81", + jar_urls = [ + "http://repo1.maven.org/maven2/com/google/code/gson/gson/2.8.5/gson-2.8.5.jar", + "http://maven.ibiblio.org/maven2/com/google/code/gson/gson/2.8.5/gson-2.8.5.jar", + ], + ) + def com_google_cloud_bigdataoss_gcsio(): java_import_external( name = "com_google_cloud_bigdataoss_gcsio", @@ -1377,12 +1413,12 @@ def com_google_gwt_user(): def com_google_http_client(): java_import_external( name = "com_google_http_client", - jar_sha256 = "f88ffa329ac52fb4f2ff0eb877ef7318423ac9b791a107f886ed5c7a00e77e11", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client/1.22.0/google-http-client-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/http-client/google-http-client/1.22.0/google-http-client-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "fb7d80a515da4618e2b402e1fef96999e07621b381a5889ef091482c5a3e961d", + jar_urls = [ + "http://repo1.maven.org/maven2/com/google/http-client/google-http-client/1.25.0/google-http-client-1.25.0.jar", + "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client/1.25.0/google-http-client-1.25.0.jar", + ], deps = [ "@com_google_code_findbugs_jsr305", "@com_google_guava", @@ -1409,12 +1445,12 @@ def com_google_http_client_appengine(): def com_google_http_client_jackson2(): java_import_external( name = "com_google_http_client_jackson2", - jar_sha256 = "45b1e34b2dcef5cb496ef25a1223d19cf102b8c2ea4abf96491631b2faf4611c", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client-jackson2/1.22.0/google-http-client-jackson2-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2/1.22.0/google-http-client-jackson2-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "f9e7e0d318860a2092d70b56331976280c4e9348a065ede3b99c92aa032fd853", + jar_urls = [ + "http://maven.ibiblio.org/maven2/com/google/http-client/google-http-client-jackson2/1.25.0/google-http-client-jackson2-1.25.0.jar", + "http://repo1.maven.org/maven2/com/google/http-client/google-http-client-jackson2/1.25.0/google-http-client-jackson2-1.25.0.jar", + ], deps = [ "@com_google_http_client", "@com_fasterxml_jackson_core", @@ -1424,12 +1460,12 @@ def com_google_http_client_jackson2(): def com_google_oauth_client(): java_import_external( name = "com_google_oauth_client", - jar_sha256 = "a4c56168b3e042105d68cf136e40e74f6e27f63ed0a948df966b332678e19022", - jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/oauth-client/google-oauth-client/1.22.0/google-oauth-client-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client/1.22.0/google-oauth-client-1.22.0.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "7e2929133d4231e702b5956a7e5dc8347a352acc1e97082b40c3585b81cd3501", + jar_urls = [ + "http://maven.ibiblio.org/maven2/com/google/oauth-client/google-oauth-client/1.25.0/google-oauth-client-1.25.0.jar", + "http://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client/1.25.0/google-oauth-client-1.25.0.jar", + ], deps = [ "@com_google_http_client", "@com_google_code_findbugs_jsr305", @@ -1458,10 +1494,10 @@ def com_google_oauth_client_java6(): java_import_external( name = "com_google_oauth_client_java6", licenses = ["notice"], # The Apache Software License, Version 2.0 - jar_sha256 = "c8d61bbb65f6721b85c38a88e4cb2a1782e04b8055589036705391361b658197", + jar_sha256 = "1065d7ec93a9ca93005e85d73f23f71353dd731f5c5f0310d66735ad81a16c33", jar_urls = [ - "http://maven.ibiblio.org/maven2/com/google/oauth-client/google-oauth-client-java6/1.22.0/google-oauth-client-java6-1.22.0.jar", - "http://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client-java6/1.22.0/google-oauth-client-java6-1.22.0.jar", + "http://maven.ibiblio.org/maven2/com/google/oauth-client/google-oauth-client-java6/1.27.0/google-oauth-client-java6-1.27.0.jar", + "http://repo1.maven.org/maven2/com/google/oauth-client/google-oauth-client-java6/1.27.0/google-oauth-client-java6-1.27.0.jar", ], deps = ["@com_google_oauth_client"], ) @@ -1496,6 +1532,17 @@ def com_google_oauth_client_servlet(): ], ) +def com_google_protobuf(): + http_archive( + name = "com_google_protobuf", + strip_prefix = "protobuf-3.6.1.3", + sha256 = "73fdad358857e120fd0fa19e071a96e15c0f23bb25f85d3f7009abfd4f264a2a", + urls = [ + "https://mirror.bazel.build/github.com/google/protobuf/archive/v3.6.1.3.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.3.tar.gz", + ], + ) + def com_google_protobuf_java(): java_import_external( name = "com_google_protobuf_java", @@ -1747,23 +1794,34 @@ def com_thoughtworks_paranamer(): def commons_codec(): java_import_external( name = "commons_codec", - jar_sha256 = "54b34e941b8e1414bd3e40d736efd3481772dc26db3296f6aa45cec9f6203d86", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "4241dfa94e711d435f29a4604a3e2de5c4aa3c165e23bd066be6fc1fc4309569", jar_urls = [ - "http://maven.ibiblio.org/maven2/commons-codec/commons-codec/1.6/commons-codec-1.6.jar", - "http://repo1.maven.org/maven2/commons-codec/commons-codec/1.6/commons-codec-1.6.jar", + "http://repo1.maven.org/maven2/commons-codec/commons-codec/1.10/commons-codec-1.10.jar", + "http://maven.ibiblio.org/maven2/commons-codec/commons-codec/1.10/commons-codec-1.10.jar", + ], + ) + +def commons_io(): + java_import_external( + name = "commons_io", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "a10418348d234968600ccb1d988efcbbd08716e1d96936ccc1880e7d22513474", + jar_urls = [ + "http://maven.ibiblio.org/maven2/commons-io/commons-io/2.5/commons-io-2.5.jar", + "http://repo1.maven.org/maven2/commons-io/commons-io/2.5/commons-io-2.5.jar", ], - licenses = ["notice"], # The Apache Software License, Version 2.0 ) def commons_logging(): java_import_external( name = "commons_logging", - jar_sha256 = "ce6f913cad1f0db3aad70186d65c5bc7ffcc9a99e3fe8e0b137312819f7c362f", - jar_urls = [ - "http://maven.ibiblio.org/maven2/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar", - "http://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar", - ], licenses = ["notice"], # The Apache Software License, Version 2.0 + jar_sha256 = "daddea1ea0be0f56978ab3006b8ac92834afeefbd9b7e4e6316fca57df0fa636", + jar_urls = [ + "http://repo1.maven.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar", + "http://maven.ibiblio.org/maven2/commons-logging/commons-logging/1.2/commons-logging-1.2.jar", + ], ) def dnsjava(): @@ -2212,12 +2270,12 @@ def org_apache_ftpserver_core(): def org_apache_httpcomponents_httpclient(): java_import_external( name = "org_apache_httpcomponents_httpclient", - jar_sha256 = "0dffc621400d6c632f55787d996b8aeca36b30746a716e079a985f24d8074057", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "7e97724443ad2a25ad8c73183431d47cc7946271bcbbdfa91a8a17522a566573", jar_urls = [ - "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar", - "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.2/httpclient-4.5.2.jar", + "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpclient/4.5.5/httpclient-4.5.5.jar", + "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpclient/4.5.5/httpclient-4.5.5.jar", ], - licenses = ["notice"], # Apache License deps = [ "@org_apache_httpcomponents_httpcore", "@commons_logging", @@ -2228,12 +2286,12 @@ def org_apache_httpcomponents_httpclient(): def org_apache_httpcomponents_httpcore(): java_import_external( name = "org_apache_httpcomponents_httpcore", - jar_sha256 = "f7bc09dc8a7003822d109634ffd3845d579d12e725ae54673e323a7ce7f5e325", + licenses = ["notice"], # Apache License, Version 2.0 + jar_sha256 = "1b4a1c0b9b4222eda70108d3c6e2befd4a6be3d9f78ff53dd7a94966fdf51fc5", jar_urls = [ - "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar", - "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.4/httpcore-4.4.4.jar", + "http://repo1.maven.org/maven2/org/apache/httpcomponents/httpcore/4.4.9/httpcore-4.4.9.jar", + "http://maven.ibiblio.org/maven2/org/apache/httpcomponents/httpcore/4.4.9/httpcore-4.4.9.jar", ], - licenses = ["notice"], # Apache License ) def org_apache_mina_core(): @@ -2570,7 +2628,7 @@ def org_yaml_snakeyaml(): def xerces_xmlParserAPIs(): java_import_external( name = "xerces_xmlParserAPIs", - licenses = ["TODO"], # NO LICENSE DECLARED + licenses = ["notice"], # Apache License, Version 2.0 jar_sha256 = "1c2867be1faa73c67e9232631241eb1df4cd0763048646e7bb575a9980e9d7e5", jar_urls = [ "http://repo1.maven.org/maven2/xerces/xmlParserAPIs/2.6.2/xmlParserAPIs-2.6.2.jar", @@ -2587,7 +2645,7 @@ def xpp3(): # http://creativecommons.org/licenses/publicdomain # Apache Software License, version 1.1 # http://www.apache.org/licenses/LICENSE-1.1 - licenses = ["TODO"], + licenses = ["notice"], jar_sha256 = "0341395a481bb887803957145a6a37879853dd625e9244c2ea2509d9bb7531b9", jar_urls = [ "http://maven.ibiblio.org/maven2/xpp3/xpp3/1.1.4c/xpp3-1.1.4c.jar", diff --git a/java/google/registry/request/BUILD b/java/google/registry/request/BUILD index 3b9eee131..870cb29f9 100644 --- a/java/google/registry/request/BUILD +++ b/java/google/registry/request/BUILD @@ -21,6 +21,7 @@ java_library( "@com_google_flogger", "@com_google_flogger_system_backend", "@com_google_guava", + "@com_google_monitoring_client_metrics", "@com_googlecode_json_simple", "@javax_inject", "@javax_servlet_api", diff --git a/java/google/registry/request/HttpException.java b/java/google/registry/request/HttpException.java index 8f39fb403..be84d6ec7 100644 --- a/java/google/registry/request/HttpException.java +++ b/java/google/registry/request/HttpException.java @@ -115,6 +115,10 @@ public abstract class HttpException extends RuntimeException { super(HttpServletResponse.SC_FORBIDDEN, message, null); } + public ForbiddenException(String message, Exception cause) { + super(HttpServletResponse.SC_FORBIDDEN, message, cause); + } + @Override public String getResponseCodeString() { return "Forbidden"; diff --git a/java/google/registry/request/JsonActionRunner.java b/java/google/registry/request/JsonActionRunner.java index 72cc26774..05c7ac22d 100644 --- a/java/google/registry/request/JsonActionRunner.java +++ b/java/google/registry/request/JsonActionRunner.java @@ -35,7 +35,7 @@ public final class JsonActionRunner { Map handleJsonRequest(Map json); } - @JsonPayload Map payload; + Map payload; JsonResponse response; @Inject public JsonActionRunner(@JsonPayload Map payload, JsonResponse response) { diff --git a/java/google/registry/request/RequestHandler.java b/java/google/registry/request/RequestHandler.java index c2748faa2..a2f8421fb 100644 --- a/java/google/registry/request/RequestHandler.java +++ b/java/google/registry/request/RequestHandler.java @@ -23,6 +23,8 @@ import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; import com.google.common.flogger.FluentLogger; import google.registry.request.auth.AuthResult; import google.registry.request.auth.RequestAuthenticator; +import google.registry.util.NonFinalForTesting; +import google.registry.util.SystemClock; import google.registry.util.TypeUtils.TypeInstantiator; import java.io.IOException; import java.util.Optional; @@ -30,6 +32,8 @@ import javax.annotation.Nullable; import javax.inject.Provider; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.joda.time.DateTime; +import org.joda.time.Duration; /** * Dagger-based request processor. @@ -64,6 +68,10 @@ public class RequestHandler { private final Router router; private final Provider> requestComponentBuilderProvider; private final RequestAuthenticator requestAuthenticator; + private final SystemClock clock = new SystemClock(); + + @NonFinalForTesting + RequestMetrics requestMetrics = new RequestMetrics(); /** * Constructor for subclasses to create a new request handler for a specific request component. @@ -143,6 +151,8 @@ public class RequestHandler { .requestModule(new RequestModule(req, rsp, authResult.get())) .build(); // Apply the selected Route to the component to produce an Action instance, and run it. + boolean success = true; + DateTime startTime = clock.nowUtc(); try { route.get().instantiator().apply(component).run(); if (route.get().action().automaticallyPrintOk()) { @@ -151,6 +161,14 @@ public class RequestHandler { } } catch (HttpException e) { e.send(rsp); + success = false; + } finally { + requestMetrics.record( + new Duration(startTime, clock.nowUtc()), + path, + method, + authResult.get().authLevel(), + success); } } } diff --git a/java/google/registry/request/RequestMetrics.java b/java/google/registry/request/RequestMetrics.java new file mode 100644 index 000000000..e7f88d954 --- /dev/null +++ b/java/google/registry/request/RequestMetrics.java @@ -0,0 +1,61 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.request; + +import static com.google.monitoring.metrics.EventMetric.DEFAULT_FITTER; + +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import com.google.monitoring.metrics.EventMetric; +import com.google.monitoring.metrics.LabelDescriptor; +import com.google.monitoring.metrics.MetricRegistryImpl; +import google.registry.request.auth.AuthLevel; +import org.joda.time.Duration; + +class RequestMetrics { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final ImmutableSet REQUEST_LABEL_DESCRIPTORS = + ImmutableSet.of( + LabelDescriptor.create("path", "target path"), + LabelDescriptor.create("method", "request method"), + LabelDescriptor.create("authLevel", "how the user was authenticated"), + LabelDescriptor.create("success", "whether the request succeeded")); + + static final EventMetric requestDurationMetric = + MetricRegistryImpl.getDefault() + .newEventMetric( + "/request/processing_time", + "Action processing time", + "milliseconds", + REQUEST_LABEL_DESCRIPTORS, + DEFAULT_FITTER); + + public RequestMetrics() {} + + public void record( + Duration duration, String path, Action.Method method, AuthLevel authLevel, boolean success) { + requestDurationMetric.record( + duration.getMillis(), + path, + String.valueOf(method), + String.valueOf(authLevel), + String.valueOf(success)); + logger.atInfo().log( + "Action called for path=%s, method=%s, authLevel=%s, success=%s. Took: %.3fs", + path, method, authLevel, success, duration.getMillis() / 1000d); + } +} diff --git a/java/google/registry/request/auth/AuthModule.java b/java/google/registry/request/auth/AuthModule.java index e2022cd7b..21d78db71 100644 --- a/java/google/registry/request/auth/AuthModule.java +++ b/java/google/registry/request/auth/AuthModule.java @@ -30,8 +30,7 @@ public class AuthModule { @Provides ImmutableList provideApiAuthenticationMechanisms( OAuthAuthenticationMechanism oauthAuthenticationMechanism) { - return ImmutableList.of( - oauthAuthenticationMechanism); + return ImmutableList.of(oauthAuthenticationMechanism); } /** Provides the OAuthService instance. */ diff --git a/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java b/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java new file mode 100644 index 000000000..c1d080730 --- /dev/null +++ b/java/google/registry/request/auth/AuthenticatedRegistrarAccessor.java @@ -0,0 +1,348 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.request.auth; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; +import static google.registry.model.ofy.ObjectifyService.ofy; + +import com.google.appengine.api.users.User; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.flogger.FluentLogger; +import dagger.Lazy; +import google.registry.config.RegistryConfig.Config; +import google.registry.groups.GroupsConnection; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.RegistrarContact; +import java.util.Optional; +import javax.annotation.concurrent.Immutable; +import javax.inject.Inject; + +/** + * Allows access only to {@link Registrar}s the current user has access to. + * + *

A user has OWNER role on a Registrar if there exists a {@link RegistrarContact} with that + * user's gaeId and the registrar as a parent. + * + *

An "admin" has in addition OWNER role on {@code #registryAdminClientId} and to all non-{@code + * REAL} registrars (see {@link Registrar#getType}). + * + *

An "admin" also has ADMIN role on ALL registrars. + * + *

A user is an "admin" if they are a GAE-admin, or if their email is in the "Support" G Suite + * group. + * + *

NOTE: to check whether the user is in the "Support" G Suite group, we need a connection to + * G Suite. This in turn requires we have valid JsonCredentials, which not all environments have set + * up. This connection will be created lazily (only if needed). + * + *

Specifically, we don't instantiate the connection if: (a) gSuiteSupportGroupEmailAddress isn't + * defined, or (b) the user is logged out, or (c) the user is a GAE-admin, or (d) bypassAdminCheck + * is true. + */ +@Immutable +public class AuthenticatedRegistrarAccessor { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + /** The role under which access is granted. */ + public enum Role { + OWNER, + ADMIN + } + + private final String userIdForLogging; + + /** + * Whether this user is an Admin, meaning either a GAE-admin or a member of the Support G Suite + * group. + */ + private final boolean isAdmin; + + /** + * Gives all roles a user has for a given clientId. + * + *

The order is significant, with "more specific to this user" coming first. + * + *

Logged out users have an empty roleMap. + */ + private final ImmutableSetMultimap roleMap; + + /** + * Bypass the "isAdmin" check making all users NOT admins. + * + *

Currently our test server doesn't let you change the user after the test server was created. + * This means we'd need multiple test files to test the same actions as both a "regular" user and + * an admin. + * + *

To overcome this - we add a flag that lets you dynamically choose whether a user is an admin + * or not by creating a fake "GAE-admin" user and then bypassing the admin check if they want to + * fake a "regular" user. + * + *

The reason we don't do it the other way around (have a flag that makes anyone an admin) is + * that such a flag would be a security risk, especially since VisibleForTesting is unenforced + * (and you could set it with reflection anyway). + * + *

Instead of having a test flag that elevates permissions (which has security concerns) we add + * this flag that reduces permissions. + */ + @VisibleForTesting public static boolean bypassAdminCheck = false; + + @Inject + public AuthenticatedRegistrarAccessor( + AuthResult authResult, + @Config("registryAdminClientId") String registryAdminClientId, + @Config("gSuiteSupportGroupEmailAddress") Optional gSuiteSupportGroupEmailAddress, + Lazy lazyGroupsConnection) { + this.isAdmin = userIsAdmin(authResult, gSuiteSupportGroupEmailAddress, lazyGroupsConnection); + + this.userIdForLogging = authResult.userIdForLogging(); + this.roleMap = createRoleMap(authResult, this.isAdmin, registryAdminClientId); + + logger.atInfo().log("%s has the following roles: %s", userIdForLogging(), roleMap); + } + + private AuthenticatedRegistrarAccessor( + String userIdForLogging, boolean isAdmin, ImmutableSetMultimap roleMap) { + this.userIdForLogging = checkNotNull(userIdForLogging); + this.roleMap = checkNotNull(roleMap); + this.isAdmin = isAdmin; + } + + /** + * Creates a "logged-in user" accessor with a given role map, used for tests. + * + *

The user will be allowed to create Registrars (and hence do OT&E setup) iff they have + * the role of ADMIN for at least one clientId. + * + *

The user's "name" in logs and exception messages is "TestUserId". + */ + @VisibleForTesting + public static AuthenticatedRegistrarAccessor createForTesting( + ImmutableSetMultimap roleMap) { + boolean isAdmin = roleMap.values().contains(Role.ADMIN); + return new AuthenticatedRegistrarAccessor("TestUserId", isAdmin, roleMap); + } + + /** + * Returns whether this user is allowed to create new Registrars and TLDs. + */ + public boolean isAdmin() { + return isAdmin; + } + + /** + * A map that gives all roles a user has for a given clientId. + * + *

Throws a {@link RegistrarAccessDeniedException} if the user is not logged in. + * + *

The result is ordered starting from "most specific to this user". + * + *

If you want to load the {@link Registrar} object from these (or any other) {@code clientId}, + * in order to perform actions on behalf of a user, you must use {@link #getRegistrar} which makes + * sure the user has permissions. + * + *

Note that this is an OPTIONAL step in the authentication - only used if we don't have any + * other clue as to the requested {@code clientId}. It is perfectly OK to get a {@code clientId} + * from any other source, as long as the registrar is then loaded using {@link #getRegistrar}. + */ + public ImmutableSetMultimap getAllClientIdWithRoles() { + return roleMap; + } + + /** + * Returns all the roles the current user has on the given registrar. + * + *

This is syntactic sugar for {@code getAllClientIdWithRoles().get(clientId)}. + */ + public ImmutableSet getRolesForRegistrar(String clientId) { + return getAllClientIdWithRoles().get(clientId); + } + + /** + * Checks if we have a given role for a given registrar. + * + *

This is syntactic sugar for {@code getAllClientIdWithRoles().containsEntry(clientId, role)}. + */ + public boolean hasRoleOnRegistrar(Role role, String clientId) { + return getAllClientIdWithRoles().containsEntry(clientId, role); + } + + /** + * "Guesses" which client ID the user wants from all those they have access to. + * + *

If no such ClientIds exist, throws a RegistrarAccessDeniedException. + * + *

This should be the ClientId "most likely wanted by the user". + * + *

If you want to load the {@link Registrar} object from this (or any other) {@code clientId}, + * in order to perform actions on behalf of a user, you must use {@link #getRegistrar} which makes + * sure the user has permissions. + * + *

Note that this is an OPTIONAL step in the authentication - only used if we don't have any + * other clue as to the requested {@code clientId}. It is perfectly OK to get a {@code clientId} + * from any other source, as long as the registrar is then loaded using {@link #getRegistrar}. + */ + public String guessClientId() throws RegistrarAccessDeniedException { + return getAllClientIdWithRoles().keySet().stream() + .findFirst() + .orElseThrow( + () -> + new RegistrarAccessDeniedException( + String.format("%s isn't associated with any registrar", userIdForLogging))); + } + + /** + * Loads a Registrar IFF the user is authorized. + * + *

Throws a {@link RegistrarAccessDeniedException} if the user is not logged in, or not + * authorized to access the requested registrar. + * + * @param clientId ID of the registrar we request + */ + public Registrar getRegistrar(String clientId) throws RegistrarAccessDeniedException { + verifyAccess(clientId); + + Registrar registrar = + Registrar.loadByClientId(clientId) + .orElseThrow( + () -> + new RegistrarAccessDeniedException( + String.format("Registrar %s not found", clientId))); + + if (!clientId.equals(registrar.getClientId())) { + logger.atSevere().log( + "registrarLoader.apply(clientId) returned a Registrar with a different clientId. " + + "Requested: %s, returned: %s.", + clientId, registrar.getClientId()); + throw new RegistrarAccessDeniedException("Internal error - please check logs"); + } + + return registrar; + } + + public void verifyAccess(String clientId) throws RegistrarAccessDeniedException { + ImmutableSet roles = getAllClientIdWithRoles().get(clientId); + + if (roles.isEmpty()) { + throw new RegistrarAccessDeniedException( + String.format("%s doesn't have access to registrar %s", userIdForLogging, clientId)); + } + logger.atInfo().log("%s has %s access to registrar %s.", userIdForLogging, roles, clientId); + } + + public String userIdForLogging() { + return userIdForLogging; + } + + @Override + public String toString() { + return toStringHelper(getClass()).add("user", userIdForLogging).toString(); + } + + private static boolean checkIsSupport( + Lazy lazyGroupsConnection, + String userEmail, + Optional gSuiteSupportGroupEmailAddress) { + if (!gSuiteSupportGroupEmailAddress.isPresent()) { + return false; + } + try { + return lazyGroupsConnection + .get() + .isMemberOfGroup(userEmail, gSuiteSupportGroupEmailAddress.get()); + } catch (RuntimeException e) { + logger.atSevere().withCause(e).log( + "Error checking whether email %s belongs to support group %s." + + " Skipping support role check", + userEmail, gSuiteSupportGroupEmailAddress); + return false; + } + } + + private static boolean userIsAdmin( + AuthResult authResult, + Optional gSuiteSupportGroupEmailAddress, + Lazy lazyGroupsConnection) { + + if (!authResult.userAuthInfo().isPresent()) { + return false; + } + + UserAuthInfo userAuthInfo = authResult.userAuthInfo().get(); + + User user = userAuthInfo.user(); + + // both GAE project admin and members of the gSuiteSupportGroupEmailAddress are considered + // admins for the RegistrarConsole. + return bypassAdminCheck + ? false + : userAuthInfo.isUserAdmin() + || checkIsSupport( + lazyGroupsConnection, user.getEmail(), gSuiteSupportGroupEmailAddress); + } + + private static ImmutableSetMultimap createRoleMap( + AuthResult authResult, + boolean isAdmin, + String registryAdminClientId) { + + if (!authResult.userAuthInfo().isPresent()) { + return ImmutableSetMultimap.of(); + } + + UserAuthInfo userAuthInfo = authResult.userAuthInfo().get(); + + User user = userAuthInfo.user(); + + ImmutableSetMultimap.Builder builder = new ImmutableSetMultimap.Builder<>(); + + logger.atInfo().log("Checking registrar contacts for user ID %s", user.getUserId()); + + ofy() + .load() + .type(RegistrarContact.class) + .filter("gaeUserId", user.getUserId()) + .forEach(contact -> builder.put(contact.getParent().getName(), Role.OWNER)); + if (isAdmin && !Strings.isNullOrEmpty(registryAdminClientId)) { + builder.put(registryAdminClientId, Role.OWNER); + } + + if (isAdmin) { + // Admins have ADMIN access to all registrars, and OWNER access to all non-REAL registrars + ofy() + .load() + .type(Registrar.class) + .forEach(registrar -> { + if (!Registrar.Type.REAL.equals(registrar.getType())) { + builder.put(registrar.getClientId(), Role.OWNER); + } + builder.put(registrar.getClientId(), Role.ADMIN); + }); + } + + return builder.build(); + } + + /** Exception thrown when the current user doesn't have access to the requested Registrar. */ + public static class RegistrarAccessDeniedException extends Exception { + RegistrarAccessDeniedException(String message) { + super(message); + } + } +} diff --git a/java/google/registry/request/auth/BUILD b/java/google/registry/request/auth/BUILD index 445110eb5..ac1ff20de 100644 --- a/java/google/registry/request/auth/BUILD +++ b/java/google/registry/request/auth/BUILD @@ -9,6 +9,8 @@ java_library( srcs = glob(["*.java"]), deps = [ "//java/google/registry/config", + "//java/google/registry/groups", + "//java/google/registry/model", "//java/google/registry/security", "@com_google_appengine_api_1_0_sdk", "@com_google_auto_value", diff --git a/java/google/registry/request/auth/LegacyAuthenticationMechanism.java b/java/google/registry/request/auth/LegacyAuthenticationMechanism.java index 6b1d3f50d..a33018af8 100644 --- a/java/google/registry/request/auth/LegacyAuthenticationMechanism.java +++ b/java/google/registry/request/auth/LegacyAuthenticationMechanism.java @@ -14,9 +14,11 @@ package google.registry.request.auth; +import static com.google.common.base.Strings.emptyToNull; import static com.google.common.base.Strings.nullToEmpty; import static google.registry.request.auth.AuthLevel.NONE; import static google.registry.request.auth.AuthLevel.USER; +import static google.registry.security.XsrfTokenManager.P_CSRF_TOKEN; import static google.registry.security.XsrfTokenManager.X_CSRF_TOKEN; import com.google.appengine.api.users.UserService; @@ -52,8 +54,7 @@ public class LegacyAuthenticationMechanism implements AuthenticationMechanism { return AuthResult.create(NONE); } - if (!SAFE_METHODS.contains(request.getMethod()) - && !xsrfTokenManager.validateToken(nullToEmpty(request.getHeader(X_CSRF_TOKEN)))) { + if (!SAFE_METHODS.contains(request.getMethod()) && !validateXsrf(request)) { return AuthResult.create(NONE); } @@ -61,4 +62,27 @@ public class LegacyAuthenticationMechanism implements AuthenticationMechanism { USER, UserAuthInfo.create(userService.getCurrentUser(), userService.isUserAdmin())); } + + private boolean validateXsrf(HttpServletRequest request) { + String headerToken = emptyToNull(request.getHeader(X_CSRF_TOKEN)); + if (headerToken != null) { + return xsrfTokenManager.validateToken(headerToken); + } + // If we got here - the header didn't have the token. + // It might be in the POST data - however even checking whether the POST data has this entry + // could break the Action! + // + // Reason: if we do request.getParameter, any Action that injects @Payload or @JsonPayload + // would break since it uses request.getReader - and it's an error to call both getReader and + // getParameter! + // + // However, in this case it's acceptable since if we got here - the POST request didn't even + // have the XSRF header meaning if it doesn't have POST data - it's not from a valid source at + // all (a valid but outdated source would have a bad header value, but getting here means we had + // no value at all) + // + // TODO(b/120201577): Once we know from the @Action whether we can use getParameter or not - + // only check getParameter if that's how this @Action uses getParameters. + return xsrfTokenManager.validateToken(nullToEmpty(request.getParameter(P_CSRF_TOKEN))); + } } diff --git a/java/google/registry/security/XsrfTokenManager.java b/java/google/registry/security/XsrfTokenManager.java index 5681ddeb2..21318fcab 100644 --- a/java/google/registry/security/XsrfTokenManager.java +++ b/java/google/registry/security/XsrfTokenManager.java @@ -37,6 +37,9 @@ public final class XsrfTokenManager { /** HTTP header used for transmitting XSRF tokens. */ public static final String X_CSRF_TOKEN = "X-CSRF-Token"; + /** POST parameter used for transmitting XSRF tokens. */ + public static final String P_CSRF_TOKEN = "xsrfToken"; + /** Maximum age of an acceptable XSRF token. */ private static final Duration XSRF_VALIDITY = Duration.standardDays(1); diff --git a/java/google/registry/tools/AllocateDomainCommand.java b/java/google/registry/tools/AllocateDomainCommand.java index 7edc31a7d..c39997d0f 100644 --- a/java/google/registry/tools/AllocateDomainCommand.java +++ b/java/google/registry/tools/AllocateDomainCommand.java @@ -21,7 +21,7 @@ import static com.google.common.base.Strings.emptyToNull; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.transform; import static com.google.common.io.BaseEncoding.base16; -import static google.registry.flows.EppXmlTransformer.unmarshal; +import static google.registry.model.eppcommon.EppXmlTransformer.unmarshal; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.tools.CommandUtilities.addHeader; import static java.util.stream.Collectors.joining; @@ -34,7 +34,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.template.soy.data.SoyMapData; import com.googlecode.objectify.Key; -import google.registry.flows.EppException; import google.registry.model.domain.DesignatedContact; import google.registry.model.domain.DomainApplication; import google.registry.model.domain.DomainCommand; @@ -46,6 +45,7 @@ import google.registry.model.eppinput.EppInput.ResourceCommandWrapper; import google.registry.model.reporting.HistoryEntry; import google.registry.model.smd.SignedMark; import google.registry.tools.soy.DomainAllocateSoyInfo; +import google.registry.xml.XmlException; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -88,7 +88,7 @@ final class AllocateDomainCommand extends MutatingEppToolCommand { } /** Extract the registration period from the XML used to create the domain application. */ - private static Period extractPeriodFromXml(byte[] xmlBytes) throws EppException { + private static Period extractPeriodFromXml(byte[] xmlBytes) throws XmlException { EppInput eppInput = unmarshal(EppInput.class, xmlBytes); return ((DomainCommand.Create) ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) @@ -182,7 +182,7 @@ final class AllocateDomainCommand extends MutatingEppToolCommand { "dsRecords", dsRecords, "clTrid", clientTransactionId)); applicationKeys.add(Key.create(application)); - } catch (EppException e) { + } catch (XmlException e) { throw new RuntimeException(e); } } diff --git a/java/google/registry/tools/AppEngineAdminApiModule.java b/java/google/registry/tools/AppEngineAdminApiModule.java new file mode 100644 index 000000000..dedbf4142 --- /dev/null +++ b/java/google/registry/tools/AppEngineAdminApiModule.java @@ -0,0 +1,39 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.util.Utils; +import com.google.api.services.appengine.v1.Appengine; +import dagger.Module; +import dagger.Provides; +import google.registry.config.CredentialModule.LocalCredential; +import google.registry.config.RegistryConfig.Config; +import javax.inject.Singleton; + +/** Module providing the instance of {@link Appengine} to access App Engine Admin Api. */ +@Module +public abstract class AppEngineAdminApiModule { + + @Provides + @Singleton + public static Appengine provideAppengine( + @LocalCredential GoogleCredential credential, @Config("projectId") String projectId) { + return new Appengine.Builder( + Utils.getDefaultTransport(), Utils.getDefaultJsonFactory(), credential) + .setApplicationName(projectId) + .build(); + } +} diff --git a/java/google/registry/tools/AppEngineConnection.java b/java/google/registry/tools/AppEngineConnection.java index 68ab47ddb..2a71272fe 100644 --- a/java/google/registry/tools/AppEngineConnection.java +++ b/java/google/registry/tools/AppEngineConnection.java @@ -66,10 +66,21 @@ class AppEngineConnection { } enum Service { - DEFAULT, - TOOLS, - BACKEND, - PUBAPI + DEFAULT("default"), + TOOLS("tools"), + BACKEND("backend"), + PUBAPI("pubapi"); + + private final String serviceId; + + Service(String serviceId) { + this.serviceId = serviceId; + } + + /** Returns the actual service id in App Engine. */ + String getServiceId() { + return serviceId; + } } /** Returns a copy of this connection that talks to a different service. */ @@ -91,8 +102,7 @@ class AppEngineConnection { private String internalSend( String endpoint, Map params, MediaType contentType, @Nullable byte[] payload) throws IOException { - GenericUrl url = new GenericUrl(getServer()); - url.setRawPath(endpoint); + GenericUrl url = new GenericUrl(String.format("%s%s", getServer(), endpoint)); url.putAll(params); HttpRequest request = (payload != null) diff --git a/java/google/registry/tools/AuthModule.java b/java/google/registry/tools/AuthModule.java index 114102d75..61286a187 100644 --- a/java/google/registry/tools/AuthModule.java +++ b/java/google/registry/tools/AuthModule.java @@ -21,39 +21,57 @@ import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInsta import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; +import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets.Details; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.util.store.AbstractDataStoreFactory; import com.google.api.client.util.store.FileDataStoreFactory; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; +import com.google.gson.Gson; +import dagger.Binds; +import dagger.Lazy; import dagger.Module; import dagger.Provides; +import google.registry.config.CredentialModule.DefaultCredential; +import google.registry.config.CredentialModule.LocalCredential; +import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.Config; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.nio.file.Files; +import java.nio.file.Paths; +import javax.annotation.Nullable; +import javax.inject.Named; import javax.inject.Qualifier; import javax.inject.Singleton; -/** - * Module providing the dependency graph for authorization credentials. - */ +/** Module providing the dependency graph for authorization credentials. */ @Module public class AuthModule { private static final File DATA_STORE_DIR = new File(System.getProperty("user.home"), ".config/nomulus/credentials"); + @Module + abstract static class LocalCredentialModule { + @Binds + @DefaultCredential + abstract GoogleCredential provideLocalCredentialAsDefaultCredential( + @LocalCredential GoogleCredential credential); + } + @Provides - public Credential provideCredential( - GoogleAuthorizationCodeFlow flow, - @ClientScopeQualifier String clientScopeQualifier) { + @StoredCredential + static Credential provideCredential( + GoogleAuthorizationCodeFlow flow, @ClientScopeQualifier String clientScopeQualifier) { try { // Try to load the credentials, throw an exception if we fail. Credential credential = flow.loadCredential(clientScopeQualifier); @@ -67,10 +85,27 @@ public class AuthModule { } @Provides - GoogleAuthorizationCodeFlow provideAuthorizationCodeFlow( + @LocalCredential + public static GoogleCredential provideLocalCredential( + @LocalCredentialJson String credentialJson, + @Config("localCredentialOauthScopes") ImmutableList scopes) { + try { + GoogleCredential credential = + GoogleCredential.fromStream(new ByteArrayInputStream(credentialJson.getBytes(UTF_8))); + if (credential.createScopedRequired()) { + credential = credential.createScoped(scopes); + } + return credential; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Provides + public static GoogleAuthorizationCodeFlow provideAuthorizationCodeFlow( JsonFactory jsonFactory, GoogleClientSecrets clientSecrets, - @Config("requiredOauthScopes") ImmutableSet requiredOauthScopes, + @Config("localCredentialOauthScopes") ImmutableList requiredOauthScopes, AbstractDataStoreFactory dataStoreFactory) { try { return new GoogleAuthorizationCodeFlow.Builder( @@ -83,41 +118,68 @@ public class AuthModule { } @Provides - AuthorizationCodeInstalledApp provideAuthorizationCodeInstalledApp( + public static AuthorizationCodeInstalledApp provideAuthorizationCodeInstalledApp( GoogleAuthorizationCodeFlow flow) { return new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()); } @Provides - GoogleClientSecrets provideClientSecrets( - @Config("clientSecretFilename") String clientSecretFilename, JsonFactory jsonFactory) { + static Details provideDefaultInstalledDetails() { + return new Details() + .setAuthUri("https://accounts.google.com/o/oauth2/auth") + .setTokenUri("https://accounts.google.com/o/oauth2/token"); + } + + @Provides + public static GoogleClientSecrets provideClientSecrets( + @Config("toolsClientId") String clientId, + @Config("toolsClientSecret") String clientSecret, + Details details) { + return new GoogleClientSecrets() + .setInstalled(details.setClientId(clientId).setClientSecret(clientSecret)); + } + + @Provides + @LocalCredentialJson + public static String provideLocalCredentialJson( + Lazy clientSecrets, + @StoredCredential Lazy credential, + @Nullable @Named("credentialFileName") String credentialFilename) { try { - // Load the client secrets file. - InputStream secretResourceStream = getClass().getResourceAsStream(clientSecretFilename); - if (secretResourceStream == null) { - throw new RuntimeException("No client secret file found: " + clientSecretFilename); + if (credentialFilename != null) { + return new String(Files.readAllBytes(Paths.get(credentialFilename)), UTF_8); + } else { + return new Gson() + .toJson( + ImmutableMap.builder() + .put("type", "authorized_user") + .put("client_id", clientSecrets.get().getDetails().getClientId()) + .put("client_secret", clientSecrets.get().getDetails().getClientSecret()) + .put("refresh_token", credential.get().getRefreshToken()) + .build()); } - return GoogleClientSecrets.load(jsonFactory, - new InputStreamReader(secretResourceStream, UTF_8)); - } catch (IOException ex) { - throw new RuntimeException(ex); + } catch (IOException e) { + throw new RuntimeException(e); } } @Provides - @OAuthClientId String provideClientId(GoogleClientSecrets clientSecrets) { + @OAuthClientId + static String provideClientId(GoogleClientSecrets clientSecrets) { return clientSecrets.getDetails().getClientId(); } @Provides - @ClientScopeQualifier String provideClientScopeQualifier( - @OAuthClientId String clientId, @Config("requiredOauthScopes") ImmutableSet scopes) { + @ClientScopeQualifier + static String provideClientScopeQualifier( + @OAuthClientId String clientId, + @Config("localCredentialOauthScopes") ImmutableList scopes) { return clientId + " " + Joiner.on(" ").join(Ordering.natural().sortedCopy(scopes)); } @Provides @Singleton - public AbstractDataStoreFactory provideDataStoreFactory() { + public static AbstractDataStoreFactory provideDataStoreFactory() { try { return new FileDataStoreFactory(DATA_STORE_DIR); } catch (IOException ex) { @@ -125,31 +187,32 @@ public class AuthModule { } } - /** Wrapper class to hold the login() function. */ - public static class Authorizer { - /** Initiate the login flow. */ - public static void login( - GoogleAuthorizationCodeFlow flow, - @ClientScopeQualifier String clientScopeQualifier) throws IOException { - new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()) - .authorize(clientScopeQualifier); - } + /** Raised when we need a user login. */ + static class LoginRequiredException extends RuntimeException { + LoginRequiredException() {} } - /** Raised when we need a user login. */ - public static class LoginRequiredException extends RuntimeException { - public LoginRequiredException() {} - } + /** + * Dagger qualifier for the {@link Credential} constructed from the data stored on disk. + * + *

This {@link Credential} should not be used in another module, hence the private qualifier. + * It's only use is to build a {@link GoogleCredential}, which is used in injection sites + * elsewhere. + */ + @Qualifier + @Documented + @Retention(RetentionPolicy.RUNTIME) + private @interface StoredCredential {} /** Dagger qualifier for the credential qualifier consisting of client and scopes. */ @Qualifier @Documented @Retention(RetentionPolicy.RUNTIME) - public @interface ClientScopeQualifier {} + @interface ClientScopeQualifier {} /** Dagger qualifier for the OAuth2 client id. */ @Qualifier @Documented @Retention(RetentionPolicy.RUNTIME) - public @interface OAuthClientId {} + @interface OAuthClientId {} } diff --git a/java/google/registry/tools/BUILD b/java/google/registry/tools/BUILD index d4671b9b9..23b5a7999 100644 --- a/java/google/registry/tools/BUILD +++ b/java/google/registry/tools/BUILD @@ -26,13 +26,14 @@ java_library( resources = glob([ "*.properties", "sql/*.sql", - - # These are example client secret files. You'll need to obtain your - # own for every environment you use and install them in this - # directory. - "resources/client_secret*.json", ]), visibility = [":allowed-tools"], + runtime_deps = [ + "//java/google/registry/module/backend", + "//java/google/registry/module/frontend", + "//java/google/registry/module/pubapi", + "//java/google/registry/module/tools", + ], deps = [ "//java/google/registry/backup", "//java/google/registry/beam/invoicing", @@ -64,10 +65,14 @@ java_library( "//java/google/registry/whois", "//java/google/registry/xjc", "//java/google/registry/xml", + "//third_party/java/jakarta_commons_io", "//third_party/jaxb", "//third_party/objectify:objectify-v4_1", "@com_beust_jcommander", "@com_google_api_client", + "@com_google_api_client_appengine", + "@com_google_api_client_java6", + "@com_google_apis_google_api_services_appengine", "@com_google_apis_google_api_services_bigquery", "@com_google_apis_google_api_services_dns", "@com_google_appengine_api_1_0_sdk", @@ -75,6 +80,7 @@ java_library( "@com_google_appengine_remote_api//:link", "@com_google_auto_value", "@com_google_code_findbugs_jsr305", + "@com_google_code_gson", "@com_google_dagger", "@com_google_flogger", "@com_google_flogger_system_backend", diff --git a/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java b/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java index 4b2bc3028..d955ff4f0 100644 --- a/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java +++ b/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java @@ -154,7 +154,7 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand { @Parameter( names = "--ip_whitelist", - description = "Comma-delimited list of IP ranges") + description = "Comma-delimited list of IP ranges. An empty string clears the whitelist.") List ipWhitelist = new ArrayList<>(); @Nullable @@ -332,7 +332,9 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand { ImmutableList.Builder ipWhitelistBuilder = new ImmutableList.Builder<>(); if (!(ipWhitelist.size() == 1 && ipWhitelist.get(0).contains("null"))) { for (String ipRange : ipWhitelist) { - ipWhitelistBuilder.add(CidrAddressBlock.create(ipRange)); + if (!ipRange.isEmpty()) { + ipWhitelistBuilder.add(CidrAddressBlock.create(ipRange)); + } } } builder.setIpAddressWhitelist(ipWhitelistBuilder.build()); diff --git a/java/google/registry/tools/CurlCommand.java b/java/google/registry/tools/CurlCommand.java index 3804b8d48..9e0e744e5 100644 --- a/java/google/registry/tools/CurlCommand.java +++ b/java/google/registry/tools/CurlCommand.java @@ -14,11 +14,15 @@ package google.registry.tools; +import static com.google.common.base.Preconditions.checkArgument; import static java.nio.charset.StandardCharsets.UTF_8; +import com.beust.jcommander.IStringConverter; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.beust.jcommander.converters.IParameterSplitter; import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.net.MediaType; @@ -50,6 +54,7 @@ class CurlCommand implements CommandWithConnection { @Parameter( names = {"-t", "--content-type"}, + converter = MediaTypeConverter.class, description = "Media type of the request body (for a POST request. Must be combined with --body)") private MediaType mimeType = MediaType.PLAIN_TEXT_UTF_8; @@ -58,6 +63,7 @@ class CurlCommand implements CommandWithConnection { // GET...) @Parameter( names = {"-d", "--data"}, + splitter = NoSplittingSplitter.class, description = "Body for a post request. If specified, a POST request is sent. If " + "absent, a GET request is sent.") @@ -95,4 +101,20 @@ class CurlCommand implements CommandWithConnection { Joiner.on("&").join(data).getBytes(UTF_8)); System.out.println(response); } + + public static class MediaTypeConverter implements IStringConverter { + @Override + public MediaType convert(String mediaType) { + List parts = Splitter.on('/').splitToList(mediaType); + checkArgument(parts.size() == 2, "invalid MediaType '%s'", mediaType); + return MediaType.create(parts.get(0), parts.get(1)).withCharset(UTF_8); + } + } + + public static class NoSplittingSplitter implements IParameterSplitter { + @Override + public List split(String value) { + return ImmutableList.of(value); + } + } } diff --git a/java/google/registry/tools/DefaultRequestFactoryModule.java b/java/google/registry/tools/DefaultRequestFactoryModule.java deleted file mode 100644 index 4f75c894e..000000000 --- a/java/google/registry/tools/DefaultRequestFactoryModule.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.javanet.NetHttpTransport; -import dagger.Binds; -import dagger.Module; -import dagger.Provides; -import google.registry.config.RegistryConfig; -import javax.inject.Named; -import javax.inject.Provider; - -/** - * Module for providing the default HttpRequestFactory. - * - * - *

This module provides a standard NetHttpTransport-based HttpRequestFactory binding. - * The binding is qualified with the name named "default" and is not consumed directly. The - * RequestFactoryModule module binds the "default" HttpRequestFactory to the unqualified - * HttpRequestFactory, allowing users to override the actual, unqualified HttpRequestFactory - * binding by replacing RequestFactoryfModule with their own module, optionally providing - * the "default" factory in some circumstances. - * - *

Localhost connections go to the App Engine dev server. The dev server differs from most HTTP - * connections in that they don't require OAuth2 credentials, but instead require a special cookie. - */ -@Module -class DefaultRequestFactoryModule { - - @Provides - @Named("default") - public HttpRequestFactory provideHttpRequestFactory( - Provider credentialProvider) { - if (RegistryConfig.areServersLocal()) { - return new NetHttpTransport() - .createRequestFactory( - request -> request - .getHeaders() - .setCookie("dev_appserver_login=test@example.com:true:1858047912411")); - } else { - return new NetHttpTransport().createRequestFactory(credentialProvider.get()); - } - } - - /** - * Module for providing HttpRequestFactory. - * - *

Localhost connections go to the App Engine dev server. The dev server differs from most HTTP - * connections in that it doesn't require OAuth2 credentials, but instead requires a special - * cookie. - */ - @Module - abstract static class RequestFactoryModule { - - @Binds - public abstract HttpRequestFactory provideHttpRequestFactory( - @Named("default") HttpRequestFactory requestFactory); - } -} diff --git a/java/google/registry/tools/DeleteAllocationTokensCommand.java b/java/google/registry/tools/DeleteAllocationTokensCommand.java index aaf02f7f1..f252616f6 100644 --- a/java/google/registry/tools/DeleteAllocationTokensCommand.java +++ b/java/google/registry/tools/DeleteAllocationTokensCommand.java @@ -101,7 +101,11 @@ final class DeleteAllocationTokensCommand extends ConfirmingCommand System.out.printf( "%s tokens: %s\n", dryRun ? "Would delete" : "Deleted", - JOINER.join(batch.stream().map(Key::getName).collect(toImmutableSet()))); + JOINER.join( + tokensToDelete.stream() + .map(AllocationToken::getToken) + .sorted() + .collect(toImmutableSet()))); return tokensToDelete.size(); } } diff --git a/java/google/registry/tools/GenerateApplicationsReportCommand.java b/java/google/registry/tools/GenerateApplicationsReportCommand.java index 5dc4b300c..afe831b36 100644 --- a/java/google/registry/tools/GenerateApplicationsReportCommand.java +++ b/java/google/registry/tools/GenerateApplicationsReportCommand.java @@ -15,7 +15,7 @@ package google.registry.tools; import static com.google.common.base.Strings.isNullOrEmpty; -import static google.registry.flows.EppXmlTransformer.unmarshal; +import static google.registry.model.eppcommon.EppXmlTransformer.unmarshal; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.model.registry.Registries.assertTldExists; import static google.registry.util.DateTimeUtils.isBeforeOrAt; @@ -29,7 +29,6 @@ import com.google.common.base.Joiner; import com.google.common.net.InternetDomainName; import com.googlecode.objectify.cmd.LoadType; import com.googlecode.objectify.cmd.Query; -import google.registry.flows.EppException; import google.registry.model.domain.DomainApplication; import google.registry.model.smd.EncodedSignedMark; import google.registry.model.smd.SignedMark; @@ -39,6 +38,7 @@ import google.registry.tmch.TmchXmlSignature; import google.registry.tools.params.PathParameter; import google.registry.util.Clock; import google.registry.util.Idn; +import google.registry.xml.XmlException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -132,7 +132,7 @@ final class GenerateApplicationsReportCommand implements CommandWithRemoteApi { SignedMark signedMark; try { signedMark = unmarshal(SignedMark.class, signedMarkData); - } catch (EppException e) { + } catch (XmlException e) { return Optional.of(makeLine(domainApplication, "Unparseable SMD")); } diff --git a/java/google/registry/tools/GetApplicationCommand.java b/java/google/registry/tools/GetApplicationCommand.java index ffa7bb6bf..e639d2d66 100644 --- a/java/google/registry/tools/GetApplicationCommand.java +++ b/java/google/registry/tools/GetApplicationCommand.java @@ -31,9 +31,7 @@ final class GetApplicationCommand extends GetEppResourceCommand { @Override public void runAndPrint() { - for (String applicationId : mainParameters) { - printResource( - "Application", applicationId, loadDomainApplication(applicationId, readTimestamp)); - } + mainParameters.forEach( + appId -> printResource("Application", appId, loadDomainApplication(appId, readTimestamp))); } } diff --git a/java/google/registry/tools/GetEppResourceCommand.java b/java/google/registry/tools/GetEppResourceCommand.java index 202b5f307..d96eb4f7a 100644 --- a/java/google/registry/tools/GetEppResourceCommand.java +++ b/java/google/registry/tools/GetEppResourceCommand.java @@ -21,7 +21,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.googlecode.objectify.Key; import google.registry.model.EppResource; -import javax.annotation.Nullable; +import java.util.Optional; import org.joda.time.DateTime; /** Abstract command to print one or more resources to stdout. */ @@ -44,17 +44,20 @@ abstract class GetEppResourceCommand implements CommandWithRemoteApi { abstract void runAndPrint(); /** - * Prints a possibly-null resource to stdout, using resourceType and uniqueId to construct a + * Prints a possibly-absent resource to stdout, using resourceType and uniqueId to construct a * nice error message if the resource was null (i.e. doesn't exist). * *

The websafe key is appended to the output for use in e.g. manual mapreduce calls. */ - void printResource(String resourceType, String uniqueId, @Nullable EppResource resource) { - System.out.println(resource != null - ? String.format("%s\n\nWebsafe key: %s", - expand ? resource.toHydratedString() : resource, - Key.create(resource).getString()) - : String.format("%s '%s' does not exist or is deleted\n", resourceType, uniqueId)); + void printResource( + String resourceType, String uniqueId, Optional resource) { + System.out.println( + resource.isPresent() + ? String.format( + "%s\n\nWebsafe key: %s", + expand ? resource.get().toHydratedString() : resource.get(), + Key.create(resource.get()).getString()) + : String.format("%s '%s' does not exist or is deleted\n", resourceType, uniqueId)); } @Override diff --git a/java/google/registry/tools/GetHostCommand.java b/java/google/registry/tools/GetHostCommand.java index 786f68923..dc1f8a195 100644 --- a/java/google/registry/tools/GetHostCommand.java +++ b/java/google/registry/tools/GetHostCommand.java @@ -32,9 +32,7 @@ final class GetHostCommand extends GetEppResourceCommand { @Override public void runAndPrint() { - for (String hostName : mainParameters) { - printResource( - "Host", hostName, loadByForeignKey(HostResource.class, hostName, readTimestamp)); - } + mainParameters.forEach( + h -> printResource("Host", h, loadByForeignKey(HostResource.class, h, readTimestamp))); } } diff --git a/java/google/registry/tools/GhostrydeCommand.java b/java/google/registry/tools/GhostrydeCommand.java index efeee8fd3..ee8807592 100644 --- a/java/google/registry/tools/GhostrydeCommand.java +++ b/java/google/registry/tools/GhostrydeCommand.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import javax.inject.Inject; import javax.inject.Provider; +import org.apache.commons.io.IOUtils; import org.bouncycastle.openpgp.PGPException; import org.bouncycastle.openpgp.PGPPrivateKey; import org.bouncycastle.openpgp.PGPPublicKey; @@ -101,9 +102,15 @@ final class GhostrydeCommand implements CommandWithRemoteApi { private void runDecrypt() throws IOException, PGPException { try (InputStream in = Files.newInputStream(input); InputStream ghostDecoder = Ghostryde.decoder(in, rdeStagingDecryptionKey.get())) { - Path outFile = - Files.isDirectory(output) ? output.resolve(input.getFileName() + ".decrypt") : output; - Files.copy(ghostDecoder, outFile, REPLACE_EXISTING); + System.err.println("output = " + output); + if (output.toString().equals("/dev/stdout")) { + System.err.println("doing copy"); + IOUtils.copy(ghostDecoder, System.out); + } else { + Path outFile = + Files.isDirectory(output) ? output.resolve(input.getFileName() + ".decrypt") : output; + Files.copy(ghostDecoder, outFile, REPLACE_EXISTING); + } } } } diff --git a/java/google/registry/tools/GtechTool.java b/java/google/registry/tools/GtechTool.java index 82b9d2157..487e76169 100644 --- a/java/google/registry/tools/GtechTool.java +++ b/java/google/registry/tools/GtechTool.java @@ -60,10 +60,13 @@ public final class GtechTool { "list_registrars", "list_tlds", "lock_domain", + "login", + "logout", "registrar_contact", "setup_ote", "uniform_rapid_suspension", "unlock_domain", + "unrenew_domain", "update_domain", "update_registrar", "update_sandbox_tld", diff --git a/java/google/registry/tools/LockDomainCommand.java b/java/google/registry/tools/LockDomainCommand.java index 5751e6170..a03b8f1a1 100644 --- a/java/google/registry/tools/LockDomainCommand.java +++ b/java/google/registry/tools/LockDomainCommand.java @@ -14,9 +14,9 @@ package google.registry.tools; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static google.registry.model.EppResourceUtils.loadByForeignKey; +import static google.registry.util.PreconditionsUtils.checkArgumentPresent; import static org.joda.time.DateTimeZone.UTC; import com.beust.jcommander.Parameters; @@ -28,6 +28,7 @@ import com.google.template.soy.data.SoyMapData; import google.registry.model.domain.DomainResource; import google.registry.model.eppcommon.StatusValue; import google.registry.tools.soy.DomainUpdateSoyInfo; +import java.util.Optional; import org.joda.time.DateTime; /** @@ -45,10 +46,11 @@ public class LockDomainCommand extends LockOrUnlockDomainCommand { // Project all domains as of the same time so that argument order doesn't affect behavior. DateTime now = DateTime.now(UTC); for (String domain : getDomains()) { - DomainResource domainResource = loadByForeignKey(DomainResource.class, domain, now); - checkArgument(domainResource != null, "Domain '%s' does not exist", domain); + Optional domainResource = loadByForeignKey(DomainResource.class, domain, now); + checkArgumentPresent(domainResource, "Domain '%s' does not exist or is deleted", domain); ImmutableSet statusesToAdd = - Sets.difference(REGISTRY_LOCK_STATUSES, domainResource.getStatusValues()).immutableCopy(); + Sets.difference(REGISTRY_LOCK_STATUSES, domainResource.get().getStatusValues()) + .immutableCopy(); if (statusesToAdd.isEmpty()) { logger.atInfo().log("Domain '%s' is already locked and needs no updates.", domain); continue; diff --git a/java/google/registry/tools/LoginCommand.java b/java/google/registry/tools/LoginCommand.java index 612d7a309..3b4fce14f 100644 --- a/java/google/registry/tools/LoginCommand.java +++ b/java/google/registry/tools/LoginCommand.java @@ -14,10 +14,12 @@ package google.registry.tools; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.extensions.java6.auth.oauth2.GooglePromptReceiver; import javax.inject.Inject; /** Authorizes the nomulus tool for OAuth 2.0 access to remote resources. */ @@ -27,9 +29,29 @@ final class LoginCommand implements Command { @Inject GoogleAuthorizationCodeFlow flow; @Inject @AuthModule.ClientScopeQualifier String clientScopeQualifier; + @Parameter( + names = "--remote", + description = + "Whether the command is run on a remote host where access to a browser is not available. " + + "If set to true, a URL will be given and a code is expected to be entered after " + + "the user completes authorization by visiting that URL.") + private boolean remote = false; + @Override public void run() throws Exception { - new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()) - .authorize(clientScopeQualifier); + AuthorizationCodeInstalledApp app; + if (remote) { + app = + new AuthorizationCodeInstalledApp( + flow, + new GooglePromptReceiver(), + url -> { + System.out.println("Please open the following address in your browser:"); + System.out.println(" " + url); + }); + } else { + app = new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()); + } + app.authorize(clientScopeQualifier); } } diff --git a/java/google/registry/tools/RegistryCli.java b/java/google/registry/tools/RegistryCli.java index 2de69bd38..2d02cb9bf 100644 --- a/java/google/registry/tools/RegistryCli.java +++ b/java/google/registry/tools/RegistryCli.java @@ -14,23 +14,27 @@ package google.registry.tools; -import com.google.appengine.tools.remoteapi.RemoteApiInstaller; -import com.google.appengine.tools.remoteapi.RemoteApiOptions; - import static com.google.common.base.Preconditions.checkState; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.tools.Injector.injectReflectively; +import static java.nio.charset.StandardCharsets.UTF_8; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; +import com.google.appengine.tools.remoteapi.RemoteApiInstaller; +import com.google.appengine.tools.remoteapi.RemoteApiOptions; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import google.registry.config.RegistryConfig; import google.registry.model.ofy.ObjectifyService; +import google.registry.tools.AuthModule.LoginRequiredException; import google.registry.tools.params.ParameterFactory; +import java.io.ByteArrayInputStream; +import java.net.URL; import java.security.Security; import java.util.Map; import org.bouncycastle.jce.provider.BouncyCastleProvider; @@ -54,6 +58,12 @@ final class RegistryCli implements AutoCloseable, CommandRunner { description = "Returns all command names.") private boolean showAllCommands; + @Parameter( + names = {"--credential"}, + description = + "Name of a JSON file containing credential information used by the tool. " + + "If not set, credentials saved by running `nomulus login' will be used.") + private String credentialJson = null; // Do not make this final - compile-time constant inlining may interfere with JCommander. @ParametersDelegate @@ -78,9 +88,6 @@ final class RegistryCli implements AutoCloseable, CommandRunner { this.commands = commands; Security.addProvider(new BouncyCastleProvider()); - - component = DaggerRegistryToolComponent.builder() - .build(); } // The > wildcard looks a little funny, but is needed so that @@ -144,6 +151,9 @@ final class RegistryCli implements AutoCloseable, CommandRunner { checkState(RegistryToolEnvironment.get() == environment, "RegistryToolEnvironment argument pre-processing kludge failed."); + component = + DaggerRegistryToolComponent.builder().credentialFilename(credentialJson).build(); + // JCommander stores sub-commands as nested JCommander objects containing a list of user objects // to be populated. Extract the subcommand by getting the JCommander wrapper and then // retrieving the first (and, by virtue of our usage, only) object from it. @@ -155,18 +165,33 @@ final class RegistryCli implements AutoCloseable, CommandRunner { try { runCommand(command); - } catch (AuthModule.LoginRequiredException ex) { - System.err.println("==================================================================="); - System.err.println("You must login using 'nomulus login' prior to running this command."); - System.err.println("==================================================================="); + } catch (RuntimeException ex) { + if (Throwables.getRootCause(ex) instanceof LoginRequiredException) { + System.err.println("==================================================================="); + System.err.println("You must login using 'nomulus login' prior to running this command."); + System.err.println("==================================================================="); + System.exit(1); + } else { + throw ex; + } } } @Override public void close() { if (installer != null) { - installer.uninstall(); - installer = null; + try { + installer.uninstall(); + installer = null; + } catch (IllegalArgumentException e) { + // There is no point throwing the error if the API is already uninstalled, which is most + // likely caused by something wrong when installing the API. That something (e. g. no + // credential found) must have already thrown an error message earlier (e. g. must run + // "nomulus login" first). This error message here is non-actionable. + if (!e.getMessage().equals("remote API is already uninstalled")) { + throw e; + } + } } } @@ -191,12 +216,13 @@ final class RegistryCli implements AutoCloseable, CommandRunner { installer = new RemoteApiInstaller(); RemoteApiOptions options = new RemoteApiOptions(); options.server( - getConnection().getServer().getHost(), getConnection().getServer().getPort()); + getConnection().getServer().getHost(), getPort(getConnection().getServer())); if (RegistryConfig.areServersLocal()) { // Use dev credentials for localhost. options.useDevelopmentServerCredential(); } else { - options.useApplicationDefaultCredential(); + RemoteApiOptionsUtil.useGoogleCredentialStream( + options, new ByteArrayInputStream(component.googleCredentialJson().getBytes(UTF_8))); } installer.install(options); } @@ -211,6 +237,10 @@ final class RegistryCli implements AutoCloseable, CommandRunner { command.run(); } + private int getPort(URL url) { + return url.getPort() == -1 ? url.getDefaultPort() : url.getPort(); + } + void setEnvironment(RegistryToolEnvironment environment) { this.environment = environment; } diff --git a/java/google/registry/tools/RegistryTool.java b/java/google/registry/tools/RegistryTool.java index ac47090d9..28acc36c9 100644 --- a/java/google/registry/tools/RegistryTool.java +++ b/java/google/registry/tools/RegistryTool.java @@ -104,9 +104,11 @@ public final class RegistryTool { .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class) .put("resave_epp_resource", ResaveEppResourceCommand.class) .put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class) + .put("set_num_instances", SetNumInstancesCommand.class) .put("setup_ote", SetupOteCommand.class) .put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class) .put("unlock_domain", UnlockDomainCommand.class) + .put("unrenew_domain", UnrenewDomainCommand.class) .put("update_application_status", UpdateApplicationStatusCommand.class) .put("update_claims_notice", UpdateClaimsNoticeCommand.class) .put("update_cursors", UpdateCursorsCommand.class) diff --git a/java/google/registry/tools/RegistryToolComponent.java b/java/google/registry/tools/RegistryToolComponent.java index 2c8363671..606cd36b9 100644 --- a/java/google/registry/tools/RegistryToolComponent.java +++ b/java/google/registry/tools/RegistryToolComponent.java @@ -14,9 +14,10 @@ package google.registry.tools; +import dagger.BindsInstance; import dagger.Component; import google.registry.bigquery.BigqueryModule; -import google.registry.config.CredentialModule; +import google.registry.config.CredentialModule.LocalCredentialJson; import google.registry.config.RegistryConfig.ConfigModule; import google.registry.dns.writer.VoidDnsWriterModule; import google.registry.dns.writer.clouddns.CloudDnsWriterModule; @@ -31,10 +32,13 @@ import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.URLFetchServiceModule; import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; +import google.registry.tools.AuthModule.LocalCredentialModule; import google.registry.util.AppEngineServiceUtilsImpl.AppEngineServiceUtilsModule; import google.registry.util.SystemClock.SystemClockModule; import google.registry.util.SystemSleeper.SystemSleeperModule; import google.registry.whois.WhoisModule; +import javax.annotation.Nullable; +import javax.inject.Named; import javax.inject.Singleton; /** @@ -46,23 +50,22 @@ import javax.inject.Singleton; @Singleton @Component( modules = { + AppEngineAdminApiModule.class, AppEngineServiceUtilsModule.class, - // TODO(b/36866706): Find a way to replace this with a command-line friendly version AuthModule.class, BigqueryModule.class, ConfigModule.class, - CredentialModule.class, + CloudDnsWriterModule.class, DatastoreServiceModule.class, DummyKeyringModule.class, - CloudDnsWriterModule.class, - DefaultRequestFactoryModule.class, - DefaultRequestFactoryModule.RequestFactoryModule.class, DnsUpdateWriterModule.class, Jackson2Module.class, KeyModule.class, KeyringModule.class, KmsModule.class, + LocalCredentialModule.class, RdeModule.class, + RequestFactoryModule.class, SystemClockModule.class, SystemSleeperModule.class, URLFetchServiceModule.class, @@ -98,8 +101,10 @@ interface RegistryToolComponent { void inject(PendingEscrowCommand command); void inject(RenewDomainCommand command); void inject(SendEscrowReportToIcannCommand command); + void inject(SetNumInstancesCommand command); void inject(SetupOteCommand command); void inject(UnlockDomainCommand command); + void inject(UnrenewDomainCommand command); void inject(UpdateCursorsCommand command); void inject(UpdateDomainCommand command); void inject(UpdateKmsKeyringCommand command); @@ -108,4 +113,15 @@ interface RegistryToolComponent { void inject(WhoisQueryCommand command); AppEngineConnection appEngineConnection(); + + @LocalCredentialJson + String googleCredentialJson(); + + @Component.Builder + interface Builder { + @BindsInstance + Builder credentialFilename(@Nullable @Named("credentialFileName") String credentialFilename); + + RegistryToolComponent build(); + } } diff --git a/java/google/registry/tools/RegistryToolEnvironment.java b/java/google/registry/tools/RegistryToolEnvironment.java index 26006167f..f2dff040d 100644 --- a/java/google/registry/tools/RegistryToolEnvironment.java +++ b/java/google/registry/tools/RegistryToolEnvironment.java @@ -17,10 +17,12 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import google.registry.config.RegistryEnvironment; +import google.registry.config.SystemPropertySetter; /** Enum of production environments, used for the {@code --environment} flag. */ enum RegistryToolEnvironment { @@ -75,12 +77,24 @@ enum RegistryToolEnvironment { return instance; } - /** Setup execution environment. Call this method before any classes are loaded. */ + /** Resets static class state to uninitialized state. */ + @VisibleForTesting + static void reset() { + instance = null; + } + + /** Sets up execution environment. Call this method before any classes are loaded. */ void setup() { + setup(SystemPropertySetter.PRODUCTION_IMPL); + } + + /** Sets up execution environment. Call this method before any classes are loaded. */ + @VisibleForTesting + void setup(SystemPropertySetter systemPropertySetter) { instance = this; - System.setProperty(RegistryEnvironment.PROPERTY, actualEnvironment.name()); + actualEnvironment.setup(systemPropertySetter); for (ImmutableMap.Entry entry : extraProperties.entrySet()) { - System.setProperty(entry.getKey(), entry.getValue()); + systemPropertySetter.setProperty(entry.getKey(), entry.getValue()); } } diff --git a/java/google/registry/tools/RemoteApiOptionsUtil.java b/java/google/registry/tools/RemoteApiOptionsUtil.java new file mode 100644 index 000000000..f6fe1f2a8 --- /dev/null +++ b/java/google/registry/tools/RemoteApiOptionsUtil.java @@ -0,0 +1,43 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.appengine.tools.remoteapi.RemoteApiOptions; +import java.io.InputStream; +import java.lang.reflect.Method; + +/** + * Provides a method to access {@link RemoteApiOptions#useGoogleCredentialStream(InputStream)}, + * which is a package private method. + * + *

This is obviously a hack, but until that method is exposed, we have to do this to set up the + * {@link RemoteApiOptions} with a JSON representing a user credential. + */ +public class RemoteApiOptionsUtil { + static RemoteApiOptions useGoogleCredentialStream(RemoteApiOptions options, InputStream stream) + throws Exception { + Method method = + options.getClass().getDeclaredMethod("useGoogleCredentialStream", InputStream.class); + checkState( + !method.isAccessible(), + "RemoteApiOptoins#useGoogleCredentialStream(InputStream) is accessible." + + " Stop using RemoteApiOptionsUtil."); + method.setAccessible(true); + method.invoke(options, stream); + return options; + } +} diff --git a/java/google/registry/tools/RenewDomainCommand.java b/java/google/registry/tools/RenewDomainCommand.java index 6cb956527..5dd3ef786 100644 --- a/java/google/registry/tools/RenewDomainCommand.java +++ b/java/google/registry/tools/RenewDomainCommand.java @@ -17,7 +17,7 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.util.CollectionUtils.findDuplicates; -import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; +import static google.registry.util.PreconditionsUtils.checkArgumentPresent; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; @@ -27,6 +27,7 @@ import google.registry.model.domain.DomainResource; import google.registry.tools.soy.RenewDomainSoyInfo; import google.registry.util.Clock; import java.util.List; +import java.util.Optional; import javax.inject.Inject; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -37,7 +38,7 @@ import org.joda.time.format.DateTimeFormatter; final class RenewDomainCommand extends MutatingEppToolCommand { @Parameter( - names = "--period", + names = {"-p", "--period"}, description = "Number of years to renew the registration for (defaults to 1).") private int period = 1; @@ -56,9 +57,11 @@ final class RenewDomainCommand extends MutatingEppToolCommand { checkArgument(period < 10, "Cannot renew domains for 10 or more years"); DateTime now = clock.nowUtc(); for (String domainName : mainParameters) { - DomainResource domain = loadByForeignKey(DomainResource.class, domainName, now); - checkArgumentNotNull(domain, "Domain '%s' does not exist or is deleted", domainName); + Optional domainOptional = + loadByForeignKey(DomainResource.class, domainName, now); + checkArgumentPresent(domainOptional, "Domain '%s' does not exist or is deleted", domainName); setSoyTemplate(RenewDomainSoyInfo.getInstance(), RenewDomainSoyInfo.RENEWDOMAIN); + DomainResource domain = domainOptional.get(); addSoyRecord( domain.getCurrentSponsorClientId(), new SoyMapData( diff --git a/java/google/registry/tools/RequestFactoryModule.java b/java/google/registry/tools/RequestFactoryModule.java new file mode 100644 index 000000000..f338e4f7f --- /dev/null +++ b/java/google/registry/tools/RequestFactoryModule.java @@ -0,0 +1,60 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.javanet.NetHttpTransport; +import dagger.Module; +import dagger.Provides; +import google.registry.config.CredentialModule.DefaultCredential; +import google.registry.config.RegistryConfig; + +/** + * Module for providing the HttpRequestFactory. + * + *

Localhost connections go to the App Engine dev server. The dev server differs from most HTTP + * connections in that they don't require OAuth2 credentials, but instead require a special cookie. + */ +@Module +class RequestFactoryModule { + + static final int REQUEST_TIMEOUT_MS = 10 * 60 * 1000; + + @Provides + static HttpRequestFactory provideHttpRequestFactory( + @DefaultCredential GoogleCredential credential) { + if (RegistryConfig.areServersLocal()) { + return new NetHttpTransport() + .createRequestFactory( + request -> + request + .getHeaders() + .setCookie("dev_appserver_login=test@example.com:true:1858047912411")); + } else { + return new NetHttpTransport() + .createRequestFactory( + request -> { + credential.initialize(request); + // GAE request times out after 10 min, so here we set the timeout to 10 min. This is + // needed to support some nomulus commands like updating premium lists that take + // a lot of time to complete. + // See https://developers.google.com/api-client-library/java/google-api-java-client/errors + request.setConnectTimeout(REQUEST_TIMEOUT_MS); + request.setReadTimeout(REQUEST_TIMEOUT_MS); + }); + } + } +} diff --git a/java/google/registry/tools/SetNumInstancesCommand.java b/java/google/registry/tools/SetNumInstancesCommand.java new file mode 100644 index 000000000..5df9c2f62 --- /dev/null +++ b/java/google/registry/tools/SetNumInstancesCommand.java @@ -0,0 +1,195 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.api.services.appengine.v1.Appengine; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import com.google.common.collect.MultimapBuilder; +import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; +import com.google.common.flogger.FluentLogger; +import google.registry.config.RegistryConfig.Config; +import google.registry.tools.AppEngineConnection.Service; +import google.registry.util.AppEngineServiceUtils; +import java.io.IOException; +import java.util.AbstractMap.SimpleEntry; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.inject.Inject; + +/** A command to set the number of instances for an App Engine service. */ +@Parameters( + separators = " =", + commandDescription = + "Set the number of instances for a given service and version. " + + "Note that this command only works for manual scaling service.") +final class SetNumInstancesCommand implements CommandWithRemoteApi { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final ImmutableSet ALL_VALID_SERVICES = + Arrays.stream(Service.values()).map(Service::name).collect(toImmutableSet()); + + private static final ImmutableSet ALL_DEPLOYED_SERVICE_IDS = + Arrays.stream(Service.values()).map(Service::getServiceId).collect(toImmutableSet()); + + // TODO(b/119629679): Use List after upgrading jcommander to latest version. + @Parameter( + names = "--services", + description = + "Comma-delimited list of App Engine services to set. " + + "Allowed values: [DEFAULT, TOOLS, BACKEND, PUBAPI]") + private List services = ImmutableList.of(); + + @Parameter( + names = "--versions", + description = + "Comma-delimited list of App Engine versions to set, e.g., canary. " + + "Cannot be set if --non_live_versions is set.") + private List versions = ImmutableList.of(); + + @Parameter( + names = "--num_instances", + description = + "The new number of instances for the given versions " + + "or for all non-live versions if --non_live_versions is set.", + required = true) + private Long numInstances; + + @Parameter( + names = "--non_live_versions", + description = "Whether to set number of instances for all non-live versions.", + arity = 1) + private Boolean nonLiveVersions = false; + + @Inject AppEngineServiceUtils appEngineServiceUtils; + @Inject Appengine appengine; + + @Inject + @Config("projectId") + String projectId; + + @Override + public void run() throws Exception { + Set invalidServiceIds = + Sets.difference(ImmutableSet.copyOf(services), ALL_VALID_SERVICES); + checkArgument(invalidServiceIds.isEmpty(), "Invalid service(s): %s", invalidServiceIds); + + Set serviceIds = + services.stream() + .map(service -> Service.valueOf(service).getServiceId()) + .collect(toImmutableSet()); + + if (nonLiveVersions) { + checkArgument(versions.isEmpty(), "--versions cannot be set if --non_live_versions is set"); + + serviceIds = serviceIds.isEmpty() ? ALL_DEPLOYED_SERVICE_IDS : serviceIds; + Multimap allLiveVersionsMap = getAllLiveVersionsMap(serviceIds); + Multimap manualScalingVersionsMap = getManualScalingVersionsMap(serviceIds); + + // Set number of instances for versions which are manual scaling and non-live + manualScalingVersionsMap.forEach( + (serviceId, versionId) -> { + if (!allLiveVersionsMap.containsEntry(serviceId, versionId)) { + setNumInstances(serviceId, versionId, numInstances); + } + }); + } else { + checkArgument(!serviceIds.isEmpty(), "Service must be specified"); + checkArgument(!versions.isEmpty(), "Version must be specified"); + checkArgument(numInstances > 0, "Number of instances must be greater than zero"); + + Multimap manualScalingVersionsMap = getManualScalingVersionsMap(serviceIds); + + for (String serviceId : serviceIds) { + for (String versionId : versions) { + checkArgument( + manualScalingVersionsMap.containsEntry(serviceId, versionId), + "Version %s of service %s is not managed through manual scaling", + versionId, + serviceId); + setNumInstances(serviceId, versionId, numInstances); + } + } + } + } + + private void setNumInstances(String service, String version, long numInstances) { + appEngineServiceUtils.setNumInstances(service, version, numInstances); + logger.atInfo().log( + "Successfully set version %s of service %s to %d instances.", + version, service, numInstances); + } + + private Multimap getAllLiveVersionsMap(Set services) { + try { + return Stream.of(appengine.apps().services().list(projectId).execute().getServices()) + .flatMap(Collection::stream) + .filter(service -> services.contains(service.getId())) + .flatMap( + service -> + // getAllocations returns only live versions or null + Stream.of(service.getSplit().getAllocations()) + .flatMap( + allocations -> + allocations.keySet().stream() + .map(versionId -> new SimpleEntry<>(service.getId(), versionId)))) + .collect( + Multimaps.toMultimap( + SimpleEntry::getKey, + SimpleEntry::getValue, + MultimapBuilder.treeKeys().arrayListValues()::build)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Multimap getManualScalingVersionsMap(Set services) { + return services.stream() + .flatMap( + serviceId -> { + try { + return Stream.of( + appengine + .apps() + .services() + .versions() + .list(projectId, serviceId) + .execute() + .getVersions()) + .flatMap(Collection::stream) + .filter(version -> version.getManualScaling() != null) + .map(version -> new SimpleEntry<>(serviceId, version.getId())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect( + Multimaps.toMultimap( + SimpleEntry::getKey, + SimpleEntry::getValue, + MultimapBuilder.treeKeys().arrayListValues()::build)); + } +} diff --git a/java/google/registry/tools/SetupOteCommand.java b/java/google/registry/tools/SetupOteCommand.java index 404ef1e88..b53bfcc7c 100644 --- a/java/google/registry/tools/SetupOteCommand.java +++ b/java/google/registry/tools/SetupOteCommand.java @@ -15,69 +15,30 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.tools.CommandUtilities.promptForYes; import static google.registry.util.X509Utils.loadCertificate; +import static java.nio.charset.StandardCharsets.US_ASCII; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSortedMap; -import com.google.re2j.Pattern; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.MoreFiles; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; -import google.registry.model.common.GaeUserIdConverter; -import google.registry.model.registrar.Registrar; -import google.registry.model.registry.Registry.TldState; +import google.registry.model.OteAccountBuilder; import google.registry.tools.params.PathParameter; +import google.registry.util.Clock; import google.registry.util.StringGenerator; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import java.util.Set; import javax.inject.Inject; -import javax.inject.Named; -import org.joda.money.CurrencyUnit; -import org.joda.money.Money; -import org.joda.time.DateTime; -import org.joda.time.Duration; /** Composite command to set up OT&E TLDs and accounts. */ @Parameters(separators = " =", commandDescription = "Set up OT&E TLDs and registrars") final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemoteApi { - // Regex: 3-14 alphanumeric characters or hyphens, the first of which must be a letter. - private static final Pattern REGISTRAR_PATTERN = Pattern.compile("^[a-z][-a-z0-9]{2,13}$"); private static final int PASSWORD_LENGTH = 16; - // Durations are short so that registrars can test with quick transfer (etc.) turnaround. - private static final Duration SHORT_ADD_GRACE_PERIOD = Duration.standardMinutes(60); - private static final Duration SHORT_REDEMPTION_GRACE_PERIOD = Duration.standardMinutes(10); - private static final Duration SHORT_PENDING_DELETE_LENGTH = Duration.standardMinutes(5); - - // Whether to prompt the user on command failures. Set to false for testing of these failures. - @VisibleForTesting - static boolean interactive = true; - - private static final ImmutableSortedMap EAP_FEE_SCHEDULE = - ImmutableSortedMap.of( - new DateTime(0), - Money.of(CurrencyUnit.USD, 0), - DateTime.parse("2018-03-01T00:00:00Z"), - Money.of(CurrencyUnit.USD, 100), - DateTime.parse("2022-03-01T00:00:00Z"), - Money.of(CurrencyUnit.USD, 0)); - - private static final String DEFAULT_PREMIUM_LIST = "default_sandbox_list"; - - @Inject - @Named("dnsWriterNames") - Set validDnsWriterNames; - @Parameter( names = {"-r", "--registrar"}, description = @@ -101,7 +62,7 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo names = {"--email"}, description = "the registrar's account to use for console access. " - + "Must be on the registry's G-Suite domain.", + + "Must be on the registry's G Suite domain.", required = true) private String email; @@ -121,252 +82,83 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo private String certHash; @Parameter( - names = {"--dns_writers"}, - description = "comma separated list of DNS writers to use on all TLDs", - required = true + names = {"--overwrite"}, + description = "whether to replace existing entities if we encounter any, instead of failing" ) - private List dnsWriters; - - @Parameter( - names = {"--premium_list"}, - description = "premium list to apply to all TLDs" - ) - private String premiumList = DEFAULT_PREMIUM_LIST; - - // TODO: (b/74079782) remove this flag once OT&E for .app is complete. - @Parameter( - names = {"--eap_only"}, - description = "whether to only create EAP TLD and registrar" - ) - private boolean eapOnly = false; + private boolean overwrite = false; @Inject @Config("base64StringGenerator") StringGenerator passwordGenerator; - /** - * Long registrar names are truncated and then have an incrementing digit appended at the end so - * that unique ROID suffixes can be generated for all TLDs for the registrar. - */ - private int roidSuffixCounter = 0; + @Inject Clock clock; - /** Runs a command, clearing the cache before and prompting the user on failures. */ - private void runCommand(Command command) { - ofy().clearSessionCache(); - try { - command.run(); - } catch (Exception e) { - System.err.format("Command failed with error %s\n", e); - if (interactive && promptForYes("Continue to next command?")) { - return; - } - Throwables.throwIfUnchecked(e); - throw new RuntimeException(e); - } - } - - /** Constructs and runs a CreateTldCommand. */ - private void createTld( - String tldName, - TldState initialTldState, - Duration addGracePeriod, - Duration redemptionGracePeriod, - Duration pendingDeleteLength, - boolean isEarlyAccess) { - CreateTldCommand command = new CreateTldCommand(); - command.addGracePeriod = addGracePeriod; - command.dnsWriters = dnsWriters; - command.validDnsWriterNames = validDnsWriterNames; - command.force = force; - command.initialTldState = initialTldState; - command.mainParameters = ImmutableList.of(tldName); - command.pendingDeleteLength = pendingDeleteLength; - command.premiumListName = Optional.of(premiumList); - String tldNameAlphaNumerical = tldName.replaceAll("[^a-z0-9]", ""); - command.roidSuffix = - String.format( - "%S%X", - tldNameAlphaNumerical.substring(0, Math.min(tldNameAlphaNumerical.length(), 7)), - roidSuffixCounter++); - command.redemptionGracePeriod = redemptionGracePeriod; - if (isEarlyAccess) { - command.eapFeeSchedule = EAP_FEE_SCHEDULE; - } - runCommand(command); - } - - /** Constructs and runs a CreateRegistrarCommand */ - private void createRegistrar(String registrarName, String password, String tld) { - CreateRegistrarCommand command = new CreateRegistrarCommand(); - command.mainParameters = ImmutableList.of(registrarName); - command.createGoogleGroups = false; // Don't create Google Groups for OT&E registrars. - command.allowedTlds = ImmutableList.of(tld); - command.registrarName = registrarName; - command.registrarType = Registrar.Type.OTE; - command.password = password; - command.clientCertificateFilename = certFile; - command.clientCertificateHash = certHash; - command.ipWhitelist = ipWhitelist; - command.street = ImmutableList.of("e-street"); - command.city = "Neverland"; - command.state = "ofmind"; - command.countryCode = "US"; - command.zip = "55555"; - command.email = Optional.of("foo@neverland.com"); - command.fax = Optional.of("+1.2125550100"); - command.phone = Optional.of("+1.2125550100"); - command.icannReferralEmail = "nightmare@registrar.test"; - command.force = force; - runCommand(command); - } - - /** Constructs and runs a RegistrarContactCommand */ - private void createRegistrarContact(String registrarName) { - RegistrarContactCommand command = new RegistrarContactCommand(); - command.mainParameters = ImmutableList.of(registrarName); - command.mode = RegistrarContactCommand.Mode.CREATE; - command.name = email; - command.email = email; - command.allowConsoleAccess = true; - command.force = force; - runCommand(command); - } + OteAccountBuilder oteAccountBuilder; + String password; /** Run any pre-execute command checks */ @Override - protected boolean checkExecutionState() throws Exception { - checkArgument( - REGISTRAR_PATTERN.matcher(registrar).matches(), - "Registrar name is invalid (see usage text for requirements)."); - - // Make sure the email is "correct" - as in it's a valid email we can convert to gaeId - // There's no need to look at the result - it'll be converted again inside - // RegistrarContactCommand. - checkNotNull( - GaeUserIdConverter.convertEmailAddressToGaeUserId(email), - "Email address %s is not associated with any GAE ID", - email); - - boolean warned = false; - if (RegistryEnvironment.get() != RegistryEnvironment.SANDBOX - && RegistryEnvironment.get() != RegistryEnvironment.UNITTEST) { - System.err.printf( - "WARNING: Running against %s environment. Are " - + "you sure you didn\'t mean to run this against sandbox (e.g. \"-e SANDBOX\")?%n", - RegistryEnvironment.get()); - warned = true; - } - - if (warned && !promptForYes("Proceed despite warnings?")) { - System.out.println("Command aborted."); - return false; - } - + protected void init() throws Exception { checkArgument( certFile == null ^ certHash == null, "Must specify exactly one of client certificate file or client certificate hash."); - // Don't wait for create_registrar to fail if it's a bad certificate file. + password = passwordGenerator.createString(PASSWORD_LENGTH); + oteAccountBuilder = + OteAccountBuilder.forClientId(registrar) + .addContact(email) + .setPassword(password) + .setIpWhitelist(ipWhitelist) + .setReplaceExisting(overwrite); + if (certFile != null) { - loadCertificate(certFile.toAbsolutePath()); + String asciiCert = MoreFiles.asCharSource(certFile, US_ASCII).read(); + // Don't wait for create_registrar to fail if it's a bad certificate file. + loadCertificate(asciiCert); + oteAccountBuilder.setCertificate(asciiCert, clock.nowUtc()); + } + + if (certHash != null) { + oteAccountBuilder.setCertificateHash(certHash); } - return true; } @Override protected String prompt() { - // Each underlying command will confirm its own operation as well, so just provide - // a summary of the steps in this command. - if (eapOnly) { - return "Creating TLD:\n" - + " " + registrar + "-eap\n" - + "Creating registrar:\n" - + " " + registrar + "-5 (access to TLD " + registrar + "-eap)\n" - + "Giving contact access to this registrar:\n" - + " " + email; - } else { - return "Creating TLDs:\n" - + " " + registrar + "-sunrise\n" - + " " + registrar + "-landrush\n" - + " " + registrar + "-ga\n" - + " " + registrar + "-eap\n" - + "Creating registrars:\n" - + " " + registrar + "-1 (access to TLD " + registrar + "-sunrise)\n" - + " " + registrar + "-2 (access to TLD " + registrar + "-landrush)\n" - + " " + registrar + "-3 (access to TLD " + registrar + "-ga)\n" - + " " + registrar + "-4 (access to TLD " + registrar + "-ga)\n" - + " " + registrar + "-5 (access to TLD " + registrar + "-eap)\n" - + "Giving contact access to these registrars:\n" - + " " + email; + ImmutableMap registrarToTldMap = oteAccountBuilder.getClientIdToTldMap(); + StringBuilder builder = new StringBuilder(); + builder.append("Creating TLDs:"); + registrarToTldMap.values().forEach(tld -> builder.append("\n ").append(tld)); + builder.append("\nCreating registrars:"); + registrarToTldMap.forEach( + (clientId, tld) -> + builder.append(String.format("\n %s (with access to %s)", clientId, tld))); + builder.append("\nGiving contact access to these registrars:").append("\n ").append(email); + + if (RegistryEnvironment.get() != RegistryEnvironment.SANDBOX + && RegistryEnvironment.get() != RegistryEnvironment.UNITTEST) { + builder.append( + String.format( + "\n\nWARNING: Running against %s environment. Are " + + "you sure you didn\'t mean to run this against sandbox (e.g. \"-e SANDBOX\")?", + RegistryEnvironment.get())); } + + return builder.toString(); } @Override public String execute() throws Exception { - if (!eapOnly) { - createTld(registrar + "-sunrise", TldState.START_DATE_SUNRISE, null, null, null, false); - createTld(registrar + "-landrush", TldState.LANDRUSH, null, null, null, false); - createTld( - registrar + "-ga", - TldState.GENERAL_AVAILABILITY, - SHORT_ADD_GRACE_PERIOD, - SHORT_REDEMPTION_GRACE_PERIOD, - SHORT_PENDING_DELETE_LENGTH, - false); - } else { - // Increase ROID suffix counter to not collide with existing TLDs. - roidSuffixCounter = roidSuffixCounter + 3; - } - createTld( - registrar + "-eap", - TldState.GENERAL_AVAILABILITY, - SHORT_ADD_GRACE_PERIOD, - SHORT_REDEMPTION_GRACE_PERIOD, - SHORT_PENDING_DELETE_LENGTH, - true); - - // Storing names and credentials in a list of tuples for later play-back. - List> registrars = new ArrayList<>(); - if (!eapOnly) { - registrars.add( - ImmutableList.of( - registrar + "-1", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-sunrise")); - registrars.add( - ImmutableList.of( - registrar + "-2", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-landrush")); - registrars.add( - ImmutableList.of( - registrar + "-3", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-ga")); - registrars.add( - ImmutableList.of( - registrar + "-4", - passwordGenerator.createString(PASSWORD_LENGTH), - registrar + "-ga")); - } - registrars.add( - ImmutableList.of( - registrar + "-5", passwordGenerator.createString(PASSWORD_LENGTH), registrar + "-eap")); - - for (List r : registrars) { - createRegistrar(r.get(0), r.get(1), r.get(2)); - createRegistrarContact(r.get(0)); - } + ImmutableMap clientIdToTld = oteAccountBuilder.buildAndPersist(); StringBuilder output = new StringBuilder(); output.append("Copy these usernames/passwords back into the onboarding bug:\n\n"); - - for (List r : registrars) { - output.append("Login: " + r.get(0) + "\n"); - output.append("Password: " + r.get(1) + "\n"); - output.append("TLD: " + r.get(2) + "\n\n"); - } + clientIdToTld.forEach( + (clientId, tld) -> { + output.append( + String.format("Login: %s\nPassword: %s\nTLD: %s\n\n", clientId, password, tld)); + }); return output.toString(); } diff --git a/java/google/registry/tools/ShellCommand.java b/java/google/registry/tools/ShellCommand.java index eb6abe680..8c57bb329 100644 --- a/java/google/registry/tools/ShellCommand.java +++ b/java/google/registry/tools/ShellCommand.java @@ -28,6 +28,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableTable; +import com.google.common.escape.Escaper; +import com.google.common.escape.Escapers; import google.registry.util.Clock; import google.registry.util.SystemClock; import java.io.BufferedReader; @@ -70,10 +72,25 @@ public class ShellCommand implements Command { private static final String ALERT_COLOR = "\u001b[1;41;97m"; // red background private static final Duration IDLE_THRESHOLD = Duration.standardHours(1); private static final String SUCCESS = "SUCCESS"; - private static final String FAILURE = "FAILURE "; + private static final String FAILURE = "FAILURE"; + private static final String RUNNING = "RUNNING"; + private static final Escaper STRING_ESCAPER = + Escapers.builder() + .addEscape('\\', "\\\\") + .addEscape('"', "\\\"") + .addEscape('\n', "\\n") + .addEscape('\r', "\\r") + .addEscape('\t', "\\t") + .build(); + /** + * The runner we received in the constructor. + * + *

We might want to update this runner based on flags (e.g. --encapsulate_output), but these + * flags aren't available in the constructor so we have to do it in the {@link #run} function. + */ + private final CommandRunner originalRunner; - private final CommandRunner runner; private final BufferedReader lineReader; private final ConsoleReader consoleReader; private final Clock clock; @@ -96,7 +113,7 @@ public class ShellCommand implements Command { boolean encapsulateOutput = false; public ShellCommand(CommandRunner runner) throws IOException { - this.runner = runner; + this.originalRunner = runner; InputStream in = System.in; if (System.console() != null) { consoleReader = new ConsoleReader(); @@ -114,7 +131,7 @@ public class ShellCommand implements Command { @VisibleForTesting ShellCommand(BufferedReader bufferedReader, Clock clock, CommandRunner runner) { - this.runner = runner; + this.originalRunner = runner; this.lineReader = bufferedReader; this.clock = clock; this.consoleReader = null; @@ -145,42 +162,11 @@ public class ShellCommand implements Command { return this; } - private static class OutputEncapsulator { - private PrintStream orgStdout; - private PrintStream orgStderr; + private static class OutputEncapsulator implements CommandRunner { + private final CommandRunner runner; - private EncapsulatingOutputStream encapsulatedOutputStream = null; - private EncapsulatingOutputStream encapsulatedErrorStream = null; - - private Exception error; - - private OutputEncapsulator() { - orgStdout = System.out; - orgStderr = System.err; - encapsulatedOutputStream = new EncapsulatingOutputStream(System.out, "out: "); - encapsulatedErrorStream = new EncapsulatingOutputStream(System.out, "err: "); - System.setOut(new PrintStream(encapsulatedOutputStream)); - System.setErr(new PrintStream(encapsulatedErrorStream)); - } - - void setError(Exception e) { - error = e; - } - - private void restoreOriginalStreams() { - try { - encapsulatedOutputStream.dumpLastLine(); - encapsulatedErrorStream.dumpLastLine(); - System.setOut(orgStdout); - System.setErr(orgStderr); - if (error != null) { - emitFailure(error); - } else { - emitSuccess(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } + private OutputEncapsulator(CommandRunner runner) { + this.runner = runner; } /** @@ -188,7 +174,7 @@ public class ShellCommand implements Command { * *

Dumps the last line of output prior to doing this. */ - private void emitSuccess() { + private static void emitSuccess() { System.out.println(SUCCESS); System.out.flush(); } @@ -198,23 +184,45 @@ public class ShellCommand implements Command { * *

Dumps the last line of output prior to doing this. */ - private void emitFailure(Throwable e) { - System.out.println( - FAILURE - + e.getClass().getName() - + " " - + e.getMessage().replace("\\", "\\\\").replace("\n", "\\n")); + private static void emitFailure(Throwable e) { + System.out.format( + "%s %s %s\n", FAILURE, e.getClass().getName(), STRING_ESCAPER.escape(e.getMessage())); + System.out.flush(); + } + + private static void emitArguments(String[] args) { + System.out.print(RUNNING); + Arrays.stream(args).forEach(arg -> System.out.format(" \"%s\"", STRING_ESCAPER.escape(arg))); + System.out.println(); + System.out.flush(); + } + + private void encapsulatedRun(String[] args) throws Exception { + PrintStream orgOut = System.out; + PrintStream orgErr = System.err; + try (PrintStream newOut = + new PrintStream(new EncapsulatingOutputStream(System.out, "out: ")); + PrintStream newErr = + new PrintStream(new EncapsulatingOutputStream(System.out, "err: "))) { + System.setOut(newOut); + System.setErr(newErr); + runner.run(args); + } finally { + System.setOut(orgOut); + System.setErr(orgErr); + } } /** Run "func" with output encapsulation. */ - static void run(CommandRunner runner, String[] args) { - OutputEncapsulator encapsulator = new OutputEncapsulator(); + @Override + public void run(String[] args) { + try { - runner.run(args); + emitArguments(args); + encapsulatedRun(args); + emitSuccess(); } catch (Exception e) { - encapsulator.setError(e); - } finally { - encapsulator.restoreOriginalStreams(); + emitFailure(e); } } } @@ -222,6 +230,10 @@ public class ShellCommand implements Command { /** Run the shell until the user presses "Ctrl-D". */ @Override public void run() { + // Wrap standard output and error if requested. We have to do so here in run because the flags + // haven't been processed in the constructor. + CommandRunner runner = + encapsulateOutput ? new OutputEncapsulator(originalRunner) : originalRunner; // On Production we want to be extra careful - to prevent accidental use. boolean beExtraCareful = (RegistryToolEnvironment.get() == RegistryToolEnvironment.PRODUCTION); setPrompt(RegistryToolEnvironment.get(), beExtraCareful); @@ -242,16 +254,11 @@ public class ShellCommand implements Command { continue; } - // Wrap standard output and error if requested. We have to do so here in run because the flags - // haven't been processed in the constructor. - if (encapsulateOutput) { - OutputEncapsulator.run(runner, lineArgs); - } else { - try { - runner.run(lineArgs); - } catch (Exception e) { - System.err.println("Got an exception:\n" + e); - } + try { + runner.run(lineArgs); + } catch (Exception e) { + System.err.println("Got an exception:\n" + e); + e.printStackTrace(); } } if (!encapsulateOutput) { @@ -566,6 +573,14 @@ public class ShellCommand implements Command { @Override public void flush() throws IOException { dumpLastLine(); + super.flush(); + } + + @Override + public void close() throws IOException { + dumpLastLine(); + // We do NOT want to call super.close as that would close the original outputStream + // (System.out) } /** Dump the accumulated last line of output, if there was one. */ diff --git a/java/google/registry/tools/UniformRapidSuspensionCommand.java b/java/google/registry/tools/UniformRapidSuspensionCommand.java index 9c0b70a0d..4a63d6434 100644 --- a/java/google/registry/tools/UniformRapidSuspensionCommand.java +++ b/java/google/registry/tools/UniformRapidSuspensionCommand.java @@ -20,6 +20,7 @@ import static com.google.common.collect.Sets.difference; import static google.registry.model.EppResourceUtils.checkResourcesExist; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.PreconditionsUtils.checkArgumentPresent; import static org.joda.time.DateTimeZone.UTC; import com.beust.jcommander.Parameter; @@ -37,6 +38,7 @@ import google.registry.tools.soy.UniformRapidSuspensionSoyInfo; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import javax.xml.bind.annotation.adapters.HexBinaryAdapter; import org.joda.time.DateTime; @@ -119,17 +121,17 @@ final class UniformRapidSuspensionCommand extends MutatingEppToolCommand { } catch (ClassCastException | ParseException e) { throw new IllegalArgumentException("Invalid --dsdata JSON", e); } - DomainResource domain = loadByForeignKey(DomainResource.class, domainName, now); - checkArgument(domain != null, "Domain '%s' does not exist", domainName); + Optional domain = loadByForeignKey(DomainResource.class, domainName, now); + checkArgumentPresent(domain, "Domain '%s' does not exist or is deleted", domainName); Set missingHosts = difference(newHostsSet, checkResourcesExist(HostResource.class, newHosts, now)); checkArgument(missingHosts.isEmpty(), "Hosts do not exist: %s", missingHosts); checkArgument( locksToPreserve.isEmpty() || undo, "Locks can only be preserved when running with --undo"); - existingNameservers = getExistingNameservers(domain); - existingLocks = getExistingLocks(domain); - existingDsData = getExistingDsData(domain); + existingNameservers = getExistingNameservers(domain.get()); + existingLocks = getExistingLocks(domain.get()); + existingDsData = getExistingDsData(domain.get()); setSoyTemplate( UniformRapidSuspensionSoyInfo.getInstance(), UniformRapidSuspensionSoyInfo.UNIFORMRAPIDSUSPENSION); diff --git a/java/google/registry/tools/UnlockDomainCommand.java b/java/google/registry/tools/UnlockDomainCommand.java index afd5baffe..4c6a48821 100644 --- a/java/google/registry/tools/UnlockDomainCommand.java +++ b/java/google/registry/tools/UnlockDomainCommand.java @@ -14,9 +14,9 @@ package google.registry.tools; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static google.registry.model.EppResourceUtils.loadByForeignKey; +import static google.registry.util.PreconditionsUtils.checkArgumentPresent; import static org.joda.time.DateTimeZone.UTC; import com.beust.jcommander.Parameters; @@ -28,6 +28,7 @@ import com.google.template.soy.data.SoyMapData; import google.registry.model.domain.DomainResource; import google.registry.model.eppcommon.StatusValue; import google.registry.tools.soy.DomainUpdateSoyInfo; +import java.util.Optional; import org.joda.time.DateTime; /** @@ -45,10 +46,10 @@ public class UnlockDomainCommand extends LockOrUnlockDomainCommand { // Project all domains as of the same time so that argument order doesn't affect behavior. DateTime now = DateTime.now(UTC); for (String domain : getDomains()) { - DomainResource domainResource = loadByForeignKey(DomainResource.class, domain, now); - checkArgument(domainResource != null, "Domain '%s' does not exist", domain); + Optional domainResource = loadByForeignKey(DomainResource.class, domain, now); + checkArgumentPresent(domainResource, "Domain '%s' does not exist or is deleted", domain); ImmutableSet statusesToRemove = - Sets.intersection(domainResource.getStatusValues(), REGISTRY_LOCK_STATUSES) + Sets.intersection(domainResource.get().getStatusValues(), REGISTRY_LOCK_STATUSES) .immutableCopy(); if (statusesToRemove.isEmpty()) { logger.atInfo().log("Domain '%s' is already unlocked and needs no updates.", domain); diff --git a/java/google/registry/tools/UnrenewDomainCommand.java b/java/google/registry/tools/UnrenewDomainCommand.java new file mode 100644 index 000000000..6f8e2d0e2 --- /dev/null +++ b/java/google/registry/tools/UnrenewDomainCommand.java @@ -0,0 +1,230 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static google.registry.flows.domain.DomainFlowUtils.newAutorenewBillingEvent; +import static google.registry.flows.domain.DomainFlowUtils.newAutorenewPollMessage; +import static google.registry.flows.domain.DomainFlowUtils.updateAutorenewRecurrenceEndTime; +import static google.registry.model.EppResourceUtils.loadByForeignKey; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.util.DateTimeUtils.isBeforeOrAt; +import static google.registry.util.DateTimeUtils.leapSafeSubtractYears; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.googlecode.objectify.Key; +import google.registry.model.billing.BillingEvent; +import google.registry.model.domain.DomainResource; +import google.registry.model.domain.Period; +import google.registry.model.domain.Period.Unit; +import google.registry.model.eppcommon.StatusValue; +import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex; +import google.registry.model.poll.PollMessage; +import google.registry.model.reporting.HistoryEntry; +import google.registry.model.reporting.HistoryEntry.Type; +import google.registry.util.Clock; +import google.registry.util.NonFinalForTesting; +import java.util.List; +import java.util.Optional; +import javax.inject.Inject; +import org.joda.time.DateTime; + +/** + * Command to unrenew a domain. + * + *

This removes years off a domain's registration period. Note that the expiration time cannot be + * set to prior than the present. Reversal of the charges for these years (if desired) must happen + * out of band, as they may already have been billed out and thus cannot and won't be reversed in + * Datastore. + */ +@Parameters(separators = " =", commandDescription = "Unrenew a domain.") +@NonFinalForTesting +class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemoteApi { + + @Parameter( + names = {"-p", "--period"}, + description = "Number of years to unrenew the registration for (defaults to 1).") + int period = 1; + + @Parameter(description = "Names of the domains to unrenew.", required = true) + List mainParameters; + + @Inject Clock clock; + + private static final ImmutableSet DISALLOWED_STATUSES = + ImmutableSet.of( + StatusValue.PENDING_TRANSFER, + StatusValue.SERVER_RENEW_PROHIBITED, + StatusValue.SERVER_UPDATE_PROHIBITED); + + @Override + protected void init() { + checkArgument(period >= 1 && period <= 9, "Period must be in the range 1-9"); + DateTime now = clock.nowUtc(); + ImmutableSet.Builder domainsNonexistentBuilder = new ImmutableSet.Builder<>(); + ImmutableSet.Builder domainsDeletingBuilder = new ImmutableSet.Builder<>(); + ImmutableMultimap.Builder domainsWithDisallowedStatusesBuilder = + new ImmutableMultimap.Builder<>(); + ImmutableMap.Builder domainsExpiringTooSoonBuilder = + new ImmutableMap.Builder<>(); + + for (String domainName : mainParameters) { + if (ofy().load().type(ForeignKeyDomainIndex.class).id(domainName).now() == null) { + domainsNonexistentBuilder.add(domainName); + continue; + } + Optional domain = loadByForeignKey(DomainResource.class, domainName, now); + if (!domain.isPresent() + || domain.get().getStatusValues().contains(StatusValue.PENDING_DELETE)) { + domainsDeletingBuilder.add(domainName); + continue; + } + domainsWithDisallowedStatusesBuilder.putAll( + domainName, Sets.intersection(domain.get().getStatusValues(), DISALLOWED_STATUSES)); + if (isBeforeOrAt( + leapSafeSubtractYears(domain.get().getRegistrationExpirationTime(), period), now)) { + domainsExpiringTooSoonBuilder.put(domainName, domain.get().getRegistrationExpirationTime()); + } + } + + ImmutableSet domainsNonexistent = domainsNonexistentBuilder.build(); + ImmutableSet domainsDeleting = domainsDeletingBuilder.build(); + ImmutableMultimap domainsWithDisallowedStatuses = + domainsWithDisallowedStatusesBuilder.build(); + ImmutableMap domainsExpiringTooSoon = domainsExpiringTooSoonBuilder.build(); + + boolean foundInvalidDomains = + !(domainsNonexistent.isEmpty() + && domainsDeleting.isEmpty() + && domainsWithDisallowedStatuses.isEmpty() + && domainsExpiringTooSoon.isEmpty()); + if (foundInvalidDomains) { + System.err.print("Found domains that cannot be unrenewed for the following reasons:\n\n"); + } + if (!domainsNonexistent.isEmpty()) { + System.err.printf("Domains that don't exist: %s\n\n", domainsNonexistent); + } + if (!domainsDeleting.isEmpty()) { + System.err.printf("Domains that are deleted or pending delete: %s\n\n", domainsDeleting); + } + if (!domainsWithDisallowedStatuses.isEmpty()) { + System.err.printf("Domains with disallowed statuses: %s\n\n", domainsWithDisallowedStatuses); + } + if (!domainsExpiringTooSoon.isEmpty()) { + System.err.printf("Domains expiring too soon: %s\n\n", domainsExpiringTooSoon); + } + checkArgument(!foundInvalidDomains, "Aborting because some domains cannot be unrewed"); + } + + @Override + protected String prompt() { + return String.format("Unrenew these domain(s) for %d years?", period); + } + + @Override + protected String execute() { + for (String domainName : mainParameters) { + ofy().transact(() -> unrenewDomain(domainName)); + System.out.printf("Unrenewed %s\n", domainName); + } + return "Successfully unrenewed all domains."; + } + + private void unrenewDomain(String domainName) { + ofy().assertInTransaction(); + DateTime now = ofy().getTransactionTime(); + Optional domainOptional = + loadByForeignKey(DomainResource.class, domainName, now); + // Transactional sanity checks on the off chance that something changed between init() running + // and here. + checkState( + domainOptional.isPresent() + && !domainOptional.get().getStatusValues().contains(StatusValue.PENDING_DELETE), + "Domain %s was deleted or is pending deletion", + domainName); + DomainResource domain = domainOptional.get(); + checkState( + Sets.intersection(domain.getStatusValues(), DISALLOWED_STATUSES).isEmpty(), + "Domain %s has prohibited status values", + domainName); + checkState( + leapSafeSubtractYears(domain.getRegistrationExpirationTime(), period).isAfter(now), + "Domain %s expires too soon", + domainName); + + DateTime newExpirationTime = + leapSafeSubtractYears(domain.getRegistrationExpirationTime(), period); + HistoryEntry historyEntry = + new HistoryEntry.Builder() + .setParent(domain) + .setModificationTime(now) + .setBySuperuser(true) + .setType(Type.SYNTHETIC) + .setClientId(domain.getCurrentSponsorClientId()) + .setReason("Domain unrenewal") + .setPeriod(Period.create(period, Unit.YEARS)) + .setRequestedByRegistrar(false) + .build(); + PollMessage oneTimePollMessage = + new PollMessage.OneTime.Builder() + .setClientId(domain.getCurrentSponsorClientId()) + .setMsg( + String.format( + "Domain %s was unrenewed by %d years; now expires at %s.", + domainName, period, newExpirationTime)) + .setParent(historyEntry) + .setEventTime(now) + .build(); + // Create a new autorenew billing event and poll message starting at the new expiration time. + BillingEvent.Recurring newAutorenewEvent = + newAutorenewBillingEvent(domain) + .setEventTime(newExpirationTime) + .setParent(historyEntry) + .build(); + PollMessage.Autorenew newAutorenewPollMessage = + newAutorenewPollMessage(domain) + .setEventTime(newExpirationTime) + .setParent(historyEntry) + .build(); + // End the old autorenew billing event and poll message now. + updateAutorenewRecurrenceEndTime(domain, now); + DomainResource newDomain = + domain + .asBuilder() + .setRegistrationExpirationTime(newExpirationTime) + .setLastEppUpdateTime(now) + .setLastEppUpdateClientId(domain.getCurrentSponsorClientId()) + .setAutorenewBillingEvent(Key.create(newAutorenewEvent)) + .setAutorenewPollMessage(Key.create(newAutorenewPollMessage)) + .build(); + // In order to do it'll need to write out a new HistoryEntry (likely of type SYNTHETIC), a new + // autorenew billing event and poll message, and a new one time poll message at the present time + // informing the registrar of this out-of-band change. + ofy() + .save() + .entities( + newDomain, + historyEntry, + oneTimePollMessage, + newAutorenewEvent, + newAutorenewPollMessage); + } +} diff --git a/java/google/registry/tools/UpdateApplicationStatusCommand.java b/java/google/registry/tools/UpdateApplicationStatusCommand.java index e34238c17..d143b50f9 100644 --- a/java/google/registry/tools/UpdateApplicationStatusCommand.java +++ b/java/google/registry/tools/UpdateApplicationStatusCommand.java @@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkState; import static google.registry.model.EppResourceUtils.loadDomainApplication; import static google.registry.model.domain.launch.ApplicationStatus.ALLOCATED; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import static google.registry.util.PreconditionsUtils.checkArgumentPresent; import com.beust.jcommander.Parameter; @@ -83,9 +82,12 @@ final class UpdateApplicationStatusCommand extends MutatingCommand { ofy().assertInTransaction(); DateTime now = ofy().getTransactionTime(); - // Load the domain application. - DomainApplication domainApplication = loadDomainApplication(applicationId, now); - checkArgumentNotNull(domainApplication, "Domain application does not exist"); + DomainApplication domainApplication = + loadDomainApplication(applicationId, now) + .orElseThrow( + () -> + new IllegalArgumentException( + "Domain application does not exist or is deleted")); // It's not an error if the application already has the intended status. We want the method // to be idempotent. diff --git a/java/google/registry/tools/UpdateClaimsNoticeCommand.java b/java/google/registry/tools/UpdateClaimsNoticeCommand.java index d00e00a80..41764e9f6 100644 --- a/java/google/registry/tools/UpdateClaimsNoticeCommand.java +++ b/java/google/registry/tools/UpdateClaimsNoticeCommand.java @@ -82,16 +82,20 @@ final class UpdateClaimsNoticeCommand implements CommandWithRemoteApi { DateTime now = ofy().getTransactionTime(); // Load the domain application. - DomainApplication domainApplication = loadDomainApplication(applicationId, now); - checkArgument(domainApplication != null, "Domain application does not exist"); + DomainApplication domainApplication = + loadDomainApplication(applicationId, now) + .orElseThrow( + () -> + new IllegalArgumentException( + "Domain application does not exist or is deleted")); // Make sure this isn't a sunrise application. checkArgument(domainApplication.getEncodedSignedMarks().isEmpty(), "Can't update claims notice on sunrise applications."); // Validate the new launch notice checksum. - String domainLabel = InternetDomainName.from(domainApplication.getFullyQualifiedDomainName()) - .parts().get(0); + String domainLabel = + InternetDomainName.from(domainApplication.getFullyQualifiedDomainName()).parts().get(0); launchNotice.validate(domainLabel); DomainApplication updatedApplication = domainApplication.asBuilder() diff --git a/java/google/registry/tools/UpdateDomainCommand.java b/java/google/registry/tools/UpdateDomainCommand.java index e8421582c..4e7d03431 100644 --- a/java/google/registry/tools/UpdateDomainCommand.java +++ b/java/google/registry/tools/UpdateDomainCommand.java @@ -19,7 +19,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.model.eppcommon.StatusValue.SERVER_UPDATE_PROHIBITED; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; +import static google.registry.util.PreconditionsUtils.checkArgumentPresent; import static org.joda.time.DateTimeZone.UTC; import com.beust.jcommander.Parameter; @@ -37,6 +37,7 @@ import google.registry.tools.soy.DomainUpdateSoyInfo; import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import org.joda.time.DateTime; @@ -172,8 +173,10 @@ final class UpdateDomainCommand extends CreateOrUpdateDomainCommand { if (!nameservers.isEmpty() || !admins.isEmpty() || !techs.isEmpty() || !statuses.isEmpty()) { DateTime now = DateTime.now(UTC); - DomainResource domainResource = loadByForeignKey(DomainResource.class, domain, now); - checkArgumentNotNull(domainResource, "Domain '%s' does not exist", domain); + Optional domainOptional = + loadByForeignKey(DomainResource.class, domain, now); + checkArgumentPresent(domainOptional, "Domain '%s' does not exist or is deleted", domain); + DomainResource domainResource = domainOptional.get(); checkArgument( !domainResource.getStatusValues().contains(SERVER_UPDATE_PROHIBITED), "The domain '%s' has status SERVER_UPDATE_PROHIBITED. Verify that you are allowed " diff --git a/java/google/registry/tools/UpdateSmdCommand.java b/java/google/registry/tools/UpdateSmdCommand.java index 6c516ae5f..ba1c417fa 100644 --- a/java/google/registry/tools/UpdateSmdCommand.java +++ b/java/google/registry/tools/UpdateSmdCommand.java @@ -84,11 +84,16 @@ final class UpdateSmdCommand implements CommandWithRemoteApi { DateTime now = ofy().getTransactionTime(); // Load the domain application. - DomainApplication domainApplication = loadDomainApplication(applicationId, now); - checkArgument(domainApplication != null, "Domain application does not exist"); + DomainApplication domainApplication = + loadDomainApplication(applicationId, now) + .orElseThrow( + () -> + new IllegalArgumentException( + "Domain application does not exist or is deleted")); // Make sure this is a sunrise application. - checkArgument(!domainApplication.getEncodedSignedMarks().isEmpty(), + checkArgument( + !domainApplication.getEncodedSignedMarks().isEmpty(), "Can't update SMD on a landrush application."); // Verify the new SMD. diff --git a/java/google/registry/tools/UserIdProvider.java b/java/google/registry/tools/UserIdProvider.java deleted file mode 100644 index a93a60b1e..000000000 --- a/java/google/registry/tools/UserIdProvider.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/* Here's an extra comment, just for the heck of it. */ - -package google.registry.tools; - - -/** Static methods to get the current user id. */ -class UserIdProvider { - - static String getTestUserId() { - return "test@example.com"; // Predefined default user for the development server. - } - - /** Pick up the username from an appropriate source. */ - static String getProdUserId() { - // TODO(b/28219927): fix tool authentication to use actual user credentials. - // For the time being, use the empty string so that for testing, requests without credentials - // can still pass the server-side XSRF token check (which will represent no user as ""). - return ""; - } -} diff --git a/java/google/registry/tools/ValidateLoginCredentialsCommand.java b/java/google/registry/tools/ValidateLoginCredentialsCommand.java index a4ac79904..b4c771217 100644 --- a/java/google/registry/tools/ValidateLoginCredentialsCommand.java +++ b/java/google/registry/tools/ValidateLoginCredentialsCommand.java @@ -78,7 +78,7 @@ final class ValidateLoginCredentialsCommand implements CommandWithRemoteApi { Registrar registrar = checkArgumentPresent( Registrar.loadByClientId(clientId), "Registrar %s not found", clientId); - new TlsCredentials(clientCertificateHash, Optional.of(clientIpAddress)) + new TlsCredentials(true, clientCertificateHash, Optional.of(clientIpAddress)) .validate(registrar, password); checkState(!registrar.getState().equals(Registrar.State.PENDING), "Account pending"); } diff --git a/java/google/registry/tools/VerifyOteCommand.java b/java/google/registry/tools/VerifyOteCommand.java index 1129c2e7e..5dadfec63 100644 --- a/java/google/registry/tools/VerifyOteCommand.java +++ b/java/google/registry/tools/VerifyOteCommand.java @@ -36,7 +36,12 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -/** Command to verify that a registrar has passed OT&E. */ +/** + * Command to verify that a registrar has passed OT&E. + * + *

Outputted stats may be truncated at the point where all tests passed to avoid unnecessarily + * loading lots of data. + */ @Parameters( separators = " =", commandDescription = "Verify passage of OT&E for specified (or all) registrars") diff --git a/java/google/registry/tools/resources/README.md b/java/google/registry/tools/resources/README.md deleted file mode 100644 index afaf8519e..000000000 --- a/java/google/registry/tools/resources/README.md +++ /dev/null @@ -1,9 +0,0 @@ - -# Adding Client Secrets - -This directory contains the client secret files needed by the `nomulus` tool to -connect to the Nomulus backend via OAuth2. Adding client secret files to this -directory is one of two steps you need to perform; the other is adding the -client id contained in the client secret file to the list of allowed ids in the -Nomulus configuration file. See the configuration documentation for more -information. diff --git a/java/google/registry/tools/resources/client_secret.json b/java/google/registry/tools/resources/client_secret.json deleted file mode 100644 index b111b0279..000000000 --- a/java/google/registry/tools/resources/client_secret.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "installed": { - "client_id":"SEE-README.md-IN_THIS_DIRECTORY.apps.googleusercontent.com", - "project_id":"your-registry-server", - "auth_uri":"https://accounts.google.com/o/oauth2/auth", - "token_uri":"https://accounts.google.com/o/oauth2/token", - "auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs", - "client_secret":"YOUR-CLIENT-SECRET", - "redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"] - } -} diff --git a/java/google/registry/tools/resources/client_secret_UNITTEST.json b/java/google/registry/tools/resources/client_secret_UNITTEST.json deleted file mode 100644 index 4df327a21..000000000 --- a/java/google/registry/tools/resources/client_secret_UNITTEST.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "installed": { - "client_id":"UNITTEST-CLIENT-ID", - "project_id":"DO NOT CHANGE", - "auth_uri":"https://accounts.google.com/o/oauth2/auth", - "token_uri":"https://accounts.google.com/o/oauth2/token", - "auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs", - "client_secret":"TBj4EcP5c0609ojiy2DIG6wE", - "redirect_uris":["urn:ietf:wg:oauth:2.0:oob","http://localhost"] - } -} diff --git a/java/google/registry/tools/server/BUILD b/java/google/registry/tools/server/BUILD index 0c2fa9cda..9687c3cab 100644 --- a/java/google/registry/tools/server/BUILD +++ b/java/google/registry/tools/server/BUILD @@ -11,7 +11,6 @@ java_library( "//java/google/registry/config", "//java/google/registry/dns", "//java/google/registry/export", - "//java/google/registry/flows", "//java/google/registry/gcs", "//java/google/registry/groups", "//java/google/registry/mapreduce", @@ -20,6 +19,7 @@ java_library( "//java/google/registry/request", "//java/google/registry/request/auth", "//java/google/registry/util", + "//java/google/registry/xml", "//third_party/objectify:objectify-v4_1", "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_tools_appengine_gcs_client", diff --git a/java/google/registry/tools/server/KillAllDomainApplicationsAction.java b/java/google/registry/tools/server/KillAllDomainApplicationsAction.java new file mode 100644 index 000000000..8b639ba09 --- /dev/null +++ b/java/google/registry/tools/server/KillAllDomainApplicationsAction.java @@ -0,0 +1,97 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.server; + +import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.request.Action.Method.POST; +import static google.registry.util.PipelineUtils.createJobPath; + +import com.google.appengine.tools.mapreduce.Mapper; +import com.google.common.collect.ImmutableList; +import com.google.common.flogger.FluentLogger; +import com.googlecode.objectify.Key; +import google.registry.mapreduce.MapreduceRunner; +import google.registry.model.domain.DomainApplication; +import google.registry.model.index.DomainApplicationIndex; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.reporting.HistoryEntry; +import google.registry.request.Action; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import javax.inject.Inject; + +/** + * Deletes all {@link DomainApplication} entities in Datastore. + * + *

This also deletes the corresponding {@link DomainApplicationIndex}, {@link EppResourceIndex}, + * and descendent {@link HistoryEntry}s. + */ +@Action(path = "/_dr/task/killAllDomainApplications", method = POST, auth = Auth.AUTH_INTERNAL_ONLY) +public class KillAllDomainApplicationsAction implements Runnable { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Inject MapreduceRunner mrRunner; + @Inject Response response; + + @Inject + KillAllDomainApplicationsAction() {} + + @Override + public void run() { + response.sendJavaScriptRedirect( + createJobPath( + mrRunner + .setJobName("Delete all domain applications and associated entities") + .setModuleName("tools") + .runMapOnly( + new KillAllDomainApplicationsMapper(), + ImmutableList.of(createEntityInput(DomainApplication.class))))); + } + + static class KillAllDomainApplicationsMapper extends Mapper { + + private static final long serialVersionUID = 2862967335000340688L; + + @Override + public void map(final DomainApplication application) { + ofy() + .transact( + () -> { + if (ofy().load().entity(application).now() == null) { + getContext().incrementCounter("applications already deleted"); + return; + } + Key applicationKey = Key.create(application); + DomainApplicationIndex dai = + ofy().load().key(DomainApplicationIndex.createKey(application)).now(); + EppResourceIndex eri = + ofy().load().entity(EppResourceIndex.create(applicationKey)).now(); + if (dai == null || eri == null) { + logger.atSevere().log( + "Missing index(es) for application %s; skipping.", applicationKey); + getContext().incrementCounter("missing indexes"); + return; + } + // Delete the application, its descendents, and the indexes. + ofy().delete().keys(ofy().load().ancestor(application).keys()); + ofy().delete().entities(dai, eri); + logger.atInfo().log("Deleted domain application %s.", applicationKey); + getContext().incrementCounter("applications deleted"); + }); + } + } +} diff --git a/java/google/registry/tools/server/VerifyOteAction.java b/java/google/registry/tools/server/VerifyOteAction.java index a1eb47aa7..94b48e413 100644 --- a/java/google/registry/tools/server/VerifyOteAction.java +++ b/java/google/registry/tools/server/VerifyOteAction.java @@ -14,43 +14,19 @@ package google.registry.tools.server; -import static com.google.common.base.Predicates.equalTo; -import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Maps.toMap; -import static google.registry.flows.EppXmlTransformer.unmarshal; -import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.CollectionUtils.isNullOrEmpty; -import static google.registry.util.DomainNameUtils.ACE_PREFIX; -import com.google.common.base.Ascii; import com.google.common.base.Joiner; -import com.google.common.base.Predicates; -import com.google.common.collect.HashMultiset; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Multiset; -import google.registry.flows.EppException; -import google.registry.model.domain.DomainCommand; -import google.registry.model.domain.fee.FeeCreateCommandExtension; -import google.registry.model.domain.launch.LaunchCreateExtension; -import google.registry.model.domain.secdns.SecDnsCreateExtension; -import google.registry.model.domain.secdns.SecDnsUpdateExtension; -import google.registry.model.eppinput.EppInput; -import google.registry.model.eppinput.EppInput.ResourceCommandWrapper; -import google.registry.model.host.HostCommand; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.reporting.HistoryEntry.Type; +import com.google.common.collect.Maps; +import google.registry.model.OteStats; +import google.registry.model.OteStats.StatType; import google.registry.request.Action; import google.registry.request.JsonActionRunner; import google.registry.request.JsonActionRunner.JsonAction; import google.registry.request.auth.Auth; -import java.util.ArrayList; -import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.inject.Inject; /** @@ -58,10 +34,9 @@ import javax.inject.Inject; * OT&E commands that have been run just previously to verification may not be picked up yet. */ @Action( - path = VerifyOteAction.PATH, - method = Action.Method.POST, - auth = Auth.AUTH_INTERNAL_OR_ADMIN -) + path = VerifyOteAction.PATH, + method = Action.Method.POST, + auth = Auth.AUTH_INTERNAL_OR_ADMIN) public class VerifyOteAction implements Runnable, JsonAction { public static final String PATH = "/_dr/admin/verifyOte"; @@ -80,239 +55,31 @@ public class VerifyOteAction implements Runnable, JsonAction { @SuppressWarnings("unchecked") public Map handleJsonRequest(Map json) { final boolean summarize = Boolean.parseBoolean((String) json.get("summarize")); - return toMap( - (List) json.get("registrars"), registrar -> checkRegistrar(registrar, summarize)); + + Map registrarResults = + toMap((List) json.get("registrars"), OteStats::getFromRegistrar); + return Maps.transformValues(registrarResults, stats -> transformOteStats(stats, summarize)); } - /** Checks whether the provided registrar has passed OT&E and returns relevant information. */ - private String checkRegistrar(String registrarName, boolean summarize) { - HistoryEntryStats historyEntryStats = - new HistoryEntryStats().recordRegistrarHistory(registrarName); - List failureMessages = historyEntryStats.findFailures(); - int testsPassed = StatType.NUM_REQUIREMENTS - failureMessages.size(); - String status = failureMessages.isEmpty() ? "PASS" : "FAIL"; + private String transformOteStats(OteStats stats, boolean summarize) { + List failures = stats.getFailures(); + int numRequiredTests = StatType.REQUIRED_STAT_TYPES.size(); + int testsPassed = numRequiredTests - failures.size(); + + String status = failures.isEmpty() ? "PASS" : "FAIL"; return summarize ? String.format( "# actions: %4d - Reqs: [%s] %2d/%2d - Overall: %s", - historyEntryStats.statCounts.size(), - historyEntryStats.toSummary(), - testsPassed, - StatType.NUM_REQUIREMENTS, - status) + stats.getSize(), getSummary(stats), testsPassed, numRequiredTests, status) : String.format( "%s\n%s\nRequirements passed: %2d/%2d\nOverall OT&E status: %s\n", - historyEntryStats, - Joiner.on('\n').join(failureMessages), - testsPassed, - StatType.NUM_REQUIREMENTS, - status); + stats, Joiner.on('\n').join(failures), testsPassed, numRequiredTests, status); } - private static final Predicate HAS_CLAIMS_NOTICE = - eppInput -> { - Optional launchCreate = - eppInput.getSingleExtension(LaunchCreateExtension.class); - return launchCreate.isPresent() && launchCreate.get().getNotice() != null; - }; - - private static final Predicate HAS_SEC_DNS = - eppInput -> - (eppInput.getSingleExtension(SecDnsCreateExtension.class).isPresent()) - || (eppInput.getSingleExtension(SecDnsUpdateExtension.class).isPresent()); - private static final Predicate IS_SUNRISE = - eppInput -> { - Optional launchCreate = - eppInput.getSingleExtension(LaunchCreateExtension.class); - return launchCreate.isPresent() && !isNullOrEmpty(launchCreate.get().getSignedMarks()); - }; - - private static final Predicate IS_IDN = - eppInput -> - ((DomainCommand.Create) - ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) - .getResourceCommand()) - .getFullyQualifiedDomainName() - .startsWith(ACE_PREFIX); - private static final Predicate IS_SUBORDINATE = - eppInput -> - !isNullOrEmpty( - ((HostCommand.Create) - ((ResourceCommandWrapper) eppInput.getCommandWrapper().getCommand()) - .getResourceCommand()) - .getInetAddresses()); - /** Enum defining the distinct statistics (types of registrar actions) to record. */ - public enum StatType { - CONTACT_CREATES(0, equalTo(Type.CONTACT_CREATE)), - CONTACT_DELETES(0, equalTo(Type.CONTACT_DELETE)), - CONTACT_TRANSFER_APPROVES(0, equalTo(Type.CONTACT_TRANSFER_APPROVE)), - CONTACT_TRANSFER_CANCELS(0, equalTo(Type.CONTACT_TRANSFER_CANCEL)), - CONTACT_TRANSFER_REJECTS(0, equalTo(Type.CONTACT_TRANSFER_REJECT)), - CONTACT_TRANSFER_REQUESTS(0, equalTo(Type.CONTACT_TRANSFER_REQUEST)), - CONTACT_UPDATES(0, equalTo(Type.CONTACT_UPDATE)), - DOMAIN_APPLICATION_CREATES(0, equalTo(Type.DOMAIN_APPLICATION_CREATE)), - DOMAIN_APPLICATION_CREATES_LANDRUSH( - 0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE.negate()), - DOMAIN_APPLICATION_CREATES_SUNRISE(0, equalTo(Type.DOMAIN_APPLICATION_CREATE), IS_SUNRISE), - DOMAIN_APPLICATION_DELETES(0, equalTo(Type.DOMAIN_APPLICATION_DELETE)), - DOMAIN_APPLICATION_UPDATES(0, equalTo(Type.DOMAIN_APPLICATION_UPDATE)), - DOMAIN_AUTORENEWS(0, equalTo(Type.DOMAIN_AUTORENEW)), - DOMAIN_CREATES(0, equalTo(Type.DOMAIN_CREATE)), - DOMAIN_CREATES_ASCII(1, equalTo(Type.DOMAIN_CREATE), IS_IDN.negate()), - DOMAIN_CREATES_IDN(1, equalTo(Type.DOMAIN_CREATE), IS_IDN), - DOMAIN_CREATES_START_DATE_SUNRISE(1, equalTo(Type.DOMAIN_CREATE), IS_SUNRISE), - DOMAIN_CREATES_WITH_CLAIMS_NOTICE(1, equalTo(Type.DOMAIN_CREATE), HAS_CLAIMS_NOTICE), - DOMAIN_CREATES_WITH_FEE( - 1, - equalTo(Type.DOMAIN_CREATE), - eppInput -> eppInput.getSingleExtension(FeeCreateCommandExtension.class).isPresent()), - DOMAIN_CREATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS), - DOMAIN_CREATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_CREATE), HAS_SEC_DNS.negate()), - DOMAIN_DELETES(2, equalTo(Type.DOMAIN_DELETE)), - DOMAIN_RENEWS(0, equalTo(Type.DOMAIN_RENEW)), - DOMAIN_RESTORES(1, equalTo(Type.DOMAIN_RESTORE)), - DOMAIN_TRANSFER_APPROVES(1, equalTo(Type.DOMAIN_TRANSFER_APPROVE)), - DOMAIN_TRANSFER_CANCELS(1, equalTo(Type.DOMAIN_TRANSFER_CANCEL)), - DOMAIN_TRANSFER_REJECTS(1, equalTo(Type.DOMAIN_TRANSFER_REJECT)), - DOMAIN_TRANSFER_REQUESTS(1, equalTo(Type.DOMAIN_TRANSFER_REQUEST)), - DOMAIN_UPDATES(0, equalTo(Type.DOMAIN_UPDATE)), - DOMAIN_UPDATES_WITH_SEC_DNS(1, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS), - DOMAIN_UPDATES_WITHOUT_SEC_DNS(0, equalTo(Type.DOMAIN_UPDATE), HAS_SEC_DNS.negate()), - HOST_CREATES(0, equalTo(Type.HOST_CREATE)), - HOST_CREATES_EXTERNAL(0, equalTo(Type.HOST_CREATE), IS_SUBORDINATE.negate()), - HOST_CREATES_SUBORDINATE(1, equalTo(Type.HOST_CREATE), IS_SUBORDINATE), - HOST_DELETES(1, equalTo(Type.HOST_DELETE)), - HOST_UPDATES(1, equalTo(Type.HOST_UPDATE)), - UNCLASSIFIED_FLOWS(0, Predicates.alwaysFalse()); - - /** The number of StatTypes with a non-zero requirement. */ - private static final int NUM_REQUIREMENTS = - (int) Stream.of(values()).filter(statType -> statType.requirement > 0).count(); - - /** Required number of times registrars must complete this action. */ - final int requirement; - - /** Filter to check the HistoryEntry Type */ - @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. - private final Predicate typeFilter; - - /** Optional filter on the EppInput. */ - @SuppressWarnings("ImmutableEnumChecker") // Predicates are immutable. - private final Optional> eppInputFilter; - - StatType(int requirement, Predicate typeFilter) { - this(requirement, typeFilter, null); - } - - StatType( - int requirement, - Predicate typeFilter, - Predicate eppInputFilter) { - this.requirement = requirement; - this.typeFilter = typeFilter; - if (eppInputFilter == null) { - this.eppInputFilter = Optional.empty(); - } else { - this.eppInputFilter = Optional.of(eppInputFilter); - } - } - - /** Returns a more human-readable translation of the enum constant. */ - String description() { - return Ascii.toLowerCase(this.name().replace('_', ' ')); - } - - /** - * Check if the {@link HistoryEntry} type matches as well as the {@link EppInput} if supplied. - */ - boolean matches(HistoryEntry.Type historyType, Optional eppInput) { - if (eppInputFilter.isPresent() && eppInput.isPresent()) { - return typeFilter.test(historyType) && eppInputFilter.get().test(eppInput.get()); - } else { - return typeFilter.test(historyType); - } - } - } - - /** Class to represent stats derived from HistoryEntry objects on actions taken by registrars. */ - static class HistoryEntryStats { - - /** Stores counts of how many times each action type was performed. */ - Multiset statCounts = HashMultiset.create(); - - /** - * Records data in the passed historyEntryStats object on what actions have been performed by - * the four numbered OT&E variants of the registrar name. - */ - HistoryEntryStats recordRegistrarHistory(String registrarName) { - ImmutableList.Builder clientIds = new ImmutableList.Builder<>(); - for (int i = 1; i <= 4; i++) { - clientIds.add(String.format("%s-%d", registrarName, i)); - } - for (HistoryEntry historyEntry : - ofy().load().type(HistoryEntry.class).filter("clientId in", clientIds.build()).list()) { - try { - record(historyEntry); - } catch (EppException e) { - throw new RuntimeException(e); - } - } - return this; - } - - /** Interprets the data in the provided HistoryEntry and increments counters. */ - void record(final HistoryEntry historyEntry) throws EppException { - byte[] xmlBytes = historyEntry.getXmlBytes(); - // xmlBytes can be null on contact create and update for safe-harbor compliance. - final Optional eppInput = - (xmlBytes == null) - ? Optional.empty() - : Optional.of(unmarshal(EppInput.class, xmlBytes)); - if (!statCounts.addAll( - EnumSet.allOf(StatType.class) - .stream() - .filter(statType -> statType.matches(historyEntry.getType(), eppInput)) - .collect(toImmutableList()))) { - statCounts.add(StatType.UNCLASSIFIED_FLOWS); - } - } - - /** - * Returns a list of failure messages describing any cases where the passed stats fail to meet - * the required thresholds, or the empty list if all requirements are met. - */ - List findFailures() { - List messages = new ArrayList<>(); - for (StatType statType : StatType.values()) { - if (statCounts.count(statType) < statType.requirement) { - messages.add( - String.format( - "Failure: %s %s found.", - (statType.requirement == 1 ? "No" : "Not enough"), statType.description())); - } - } - return messages; - } - - /** Returns a string showing all possible actions and how many times each was performed. */ - @Override - public String toString() { - return String.format( - "%s\nTOTAL: %d", - EnumSet.allOf(StatType.class) - .stream() - .map(stat -> String.format("%s: %d", stat.description(), statCounts.count(stat))) - .collect(Collectors.joining("\n")), - statCounts.size()); - } - - /** Returns a string showing the results of each test, one character per test. */ - String toSummary() { - return EnumSet.allOf(StatType.class) - .stream() - .filter(statType -> statType.requirement > 0) - .sorted() - .map(statType -> (statCounts.count(statType) < statType.requirement) ? "." : "-") - .collect(Collectors.joining("")); - } + private String getSummary(OteStats stats) { + return StatType.REQUIRED_STAT_TYPES.stream() + .sorted() + .map(statType -> (stats.getCount(statType) < statType.getRequirement()) ? "." : "-") + .collect(Collectors.joining("")); } } diff --git a/java/google/registry/ui/css/BUILD b/java/google/registry/ui/css/BUILD index 4f02ea3ee..fc479cf07 100644 --- a/java/google/registry/ui/css/BUILD +++ b/java/google/registry/ui/css/BUILD @@ -24,6 +24,7 @@ closure_css_library( closure_css_library( name = "registrar_lib", srcs = [ + "admin-settings.css", "contact-settings.css", "contact-us.css", "dashboard.css", diff --git a/java/google/registry/ui/css/admin-settings.css b/java/google/registry/ui/css/admin-settings.css new file mode 100644 index 000000000..52df054af --- /dev/null +++ b/java/google/registry/ui/css/admin-settings.css @@ -0,0 +1,58 @@ +/** Admin Settings */ + +div#tlds div.tld { + width: 209px; +} + +#newTld { + width: 187px; + margin-left: 0.5em; +} + +div#tlds div.tld input, +div#tlds div.tld button[type=button] { + height: 27px; + line-height: 27px; + background: #ebebeb; + vertical-align: top; + border: none; + border-bottom: solid 3px white; +} + +div#tlds div.tld input { + width: 169px; + margin: 0; + padding: 0; + color: #555; + padding-left: 5px ! important; +} + +div#tlds.editing div.tld input[readonly] { + margin-left: 0.5em; +} + +div#tlds.editing div.tld button[type=button] { + display: inline-block; + float: right; + margin-left: -2px; + width: 30px; + min-width: 30px; + height: 30px; + color: grey; + font-size: 1.1em; +} + +div#tlds.editing div.tld button[type=button]:hover { + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} + +div#tlds.editing div.tld button[type=button] i { + font-style: normal; +} + +div#tlds.editing .kd-errormessage { + margin-left: 0.5em; +} + diff --git a/java/google/registry/ui/css/forms.css b/java/google/registry/ui/css/forms.css index f1844a4fa..29c5ca559 100644 --- a/java/google/registry/ui/css/forms.css +++ b/java/google/registry/ui/css/forms.css @@ -54,6 +54,11 @@ tr.subsection h3::first-letter { width: 170px; } +.description ol { + list-style-type: decimal; + margin-left: 2em; +} + /* Setting groups and labels. */ diff --git a/java/google/registry/ui/css/kd_components.css b/java/google/registry/ui/css/kd_components.css index ea44f760e..4e150c4ec 100644 --- a/java/google/registry/ui/css/kd_components.css +++ b/java/google/registry/ui/css/kd_components.css @@ -1244,12 +1244,12 @@ li.kd-labellistitem a.kd-label-blue { Component: App bars ------------------------------------------------------------------*/ .kd-appbar { - position:relative; + position: relative; padding: 21px 0; border-bottom: 1px solid #ebebeb; - height: 29px; - z-index:20; - background:#fff; + height: 66px; + z-index: 20; + background: #fff; } .kd-appbar .kd-appname { width: 160px; @@ -1263,6 +1263,22 @@ Component: App bars color: #666; white-space:nowrap; } +.kd-appbar .kd-description { + height: 29px; + font-size: 20px; + font-weight: normal; + line-height: 29px; + padding-bottom: 8px; + color: #666; + white-space:nowrap; +} +.kd-appbar .kd-description .kd-value { + color: black; + font-weight: bold; +} +.kd-appbar .kd-description form { + display: inline; +} .kd-appbar .kd-appname a { color: #666; cursor:pointer; } #stickers .kd-appbar .kd-buttonbar { margin-bottom: 0; diff --git a/java/google/registry/ui/css/registry.css b/java/google/registry/ui/css/registry.css index 9d7bbd3cd..51d668adc 100644 --- a/java/google/registry/ui/css/registry.css +++ b/java/google/registry/ui/css/registry.css @@ -22,7 +22,7 @@ h1 { } pre { - font-family: monospace; + font-family: "Courier New", Courier, monospace; white-space: pre-wrap; } @@ -48,8 +48,8 @@ input[readonly], textarea[readonly] { } textarea { - margin: 1em 0; - font-family: monospace; + margin-bottom: 1em; + font-family: "Courier New", Courier, monospace; font-size: 13px; border: solid 1px #ddd; } @@ -196,13 +196,10 @@ li.kd-menulistitem { } .kd-appbar { - padding: 0.75em 0; -} - -#reg-app-buttons { - /* Same as in reg-content below. Lines the left edge of the - appbuttons and content area with the 'R' in Registry. */ + /* same as in reg-content below. lines the left edge of the + appbuttons and content area with the 'r' in registry. */ padding-left: 173px; + padding-top: .75em; } .kd-content-sidebar { @@ -214,7 +211,7 @@ li.kd-menulistitem { #reg-nav { position: fixed; left: 0; - top: 128px; + top: 136px; width: 155px; margin: 0 25px 0 0; z-index: 3; @@ -261,7 +258,7 @@ li.kd-menulistitem { #reg-content-and-footer { position: absolute; - top: 105px; + top: 136px; left: 173px; bottom: 0; width: 100%; diff --git a/java/google/registry/ui/externs/json.js b/java/google/registry/ui/externs/json.js index b5624949a..7f99b66a8 100644 --- a/java/google/registry/ui/externs/json.js +++ b/java/google/registry/ui/externs/json.js @@ -68,6 +68,7 @@ registry.json.Response.prototype.results; // XXX: Might not need undefineds here. /** * @typedef {{ + * allowedTlds: !Array, * clientIdentifier: string, * clientCertificate: string?, * clientCertificateHash: string?, diff --git a/java/google/registry/ui/js/edit_item.js b/java/google/registry/ui/js/edit_item.js index 433646369..131f88eb7 100644 --- a/java/google/registry/ui/js/edit_item.js +++ b/java/google/registry/ui/js/edit_item.js @@ -31,10 +31,11 @@ goog.forwardDeclare('registry.Console'); * An editable item, with Edit and Save/Cancel buttons in the appbar. * @param {!registry.Console} cons * @param {function()} itemTmpl + * @param {boolean} isEditable * @constructor * @extends {registry.Component} */ -registry.EditItem = function(cons, itemTmpl) { +registry.EditItem = function(cons, itemTmpl, isEditable) { registry.EditItem.base(this, 'constructor', cons); /** @@ -48,6 +49,12 @@ registry.EditItem = function(cons, itemTmpl) { */ this.id = null; + /** + * Should the "edit" button be enabled? + * @type {boolean} + */ + this.isEditable = isEditable; + /** * Transitional id for next resource during create. * @type {?string} @@ -68,7 +75,7 @@ registry.EditItem.prototype.bindToDom = function(id) { /** Setup appbar save/edit buttons. */ registry.EditItem.prototype.setupAppbar = function() { - goog.soy.renderElement(goog.dom.getRequiredElement('reg-appbar'), + goog.soy.renderElement(goog.dom.getRequiredElement('reg-app-buttons'), registry.soy.console.appbarButtons); goog.events.listen(goog.dom.getRequiredElement('reg-app-btn-add'), goog.events.EventType.CLICK, @@ -85,11 +92,11 @@ registry.EditItem.prototype.setupAppbar = function() { goog.events.listen(goog.dom.getRequiredElement('reg-app-btn-back'), goog.events.EventType.CLICK, goog.bind(this.back, this)); - if (this.id) { - registry.util.setVisible('reg-app-btns-edit', true); - } else { - registry.util.setVisible('reg-app-btn-add', true); - } + // Show the add/edit buttons only if isEditable. + // "edit" is shown if we have an item's ID + // "add" is shown if we don't have an item's ID + registry.util.setVisible('reg-app-btns-edit', this.isEditable && !!this.id); + registry.util.setVisible('reg-app-btn-add', this.isEditable && !this.id); }; diff --git a/java/google/registry/ui/js/registrar/admin_settings.js b/java/google/registry/ui/js/registrar/admin_settings.js new file mode 100644 index 000000000..24a4c0272 --- /dev/null +++ b/java/google/registry/ui/js/registrar/admin_settings.js @@ -0,0 +1,106 @@ +// Copyright 2017 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +goog.provide('registry.registrar.AdminSettings'); + +goog.require('goog.array'); +goog.require('goog.dom'); +goog.require('goog.dom.classlist'); +goog.require('goog.events'); +goog.require('goog.events.EventType'); +goog.require('goog.soy'); +goog.require('registry.Resource'); +goog.require('registry.ResourceComponent'); +goog.require('registry.soy.registrar.admin'); + +goog.forwardDeclare('registry.registrar.Console'); + + + +/** + * Admin Settings page, such as allowed TLDs for this registrar. + * @param {!registry.registrar.Console} console + * @param {!registry.Resource} resource the RESTful resource for the registrar. + * @constructor + * @extends {registry.ResourceComponent} + * @final + */ +registry.registrar.AdminSettings = function(console, resource) { + registry.registrar.AdminSettings.base( + this, 'constructor', console, resource, + registry.soy.registrar.admin.settings, console.params.isAdmin, null); +}; +goog.inherits(registry.registrar.AdminSettings, registry.ResourceComponent); + + +/** @override */ +registry.registrar.AdminSettings.prototype.bindToDom = function(id) { + registry.registrar.AdminSettings.base(this, 'bindToDom', 'fake'); + goog.dom.removeNode(goog.dom.getRequiredElement('reg-app-btn-back')); +}; + + +/** @override */ +registry.registrar.AdminSettings.prototype.setupEditor = + function(objArgs) { + goog.dom.classlist.add(goog.dom.getRequiredElement('tlds'), + goog.getCssName('editing')); + var tlds = goog.dom.getElementsByClass(goog.getCssName('tld'), + goog.dom.getRequiredElement('tlds')); + goog.array.forEach(tlds, function(tld) { + var remBtn = goog.dom.getChildren(tld)[0]; + goog.events.listen(remBtn, + goog.events.EventType.CLICK, + goog.bind(this.onTldRemove_, this, remBtn)); + }, this); + this.typeCounts['reg-tlds'] = objArgs.allowedTlds ? + objArgs.allowedTlds.length : 0; + + goog.events.listen(goog.dom.getRequiredElement('btn-add-tld'), + goog.events.EventType.CLICK, + this.onTldAdd_, + false, + this); +}; + + +/** + * Click handler for TLD add button. + * @private + */ +registry.registrar.AdminSettings.prototype.onTldAdd_ = function() { + const tldInputElt = goog.dom.getRequiredElement('newTld'); + const tldElt = goog.soy.renderAsFragment(registry.soy.registrar.admin.tld, { + name: 'allowedTlds[' + this.typeCounts['reg-tlds'] + ']', + tld: tldInputElt.value, + }); + goog.dom.appendChild(goog.dom.getRequiredElement('tlds'), tldElt); + var remBtn = goog.dom.getFirstElementChild(tldElt); + goog.dom.classlist.remove(remBtn, goog.getCssName('hidden')); + goog.events.listen(remBtn, goog.events.EventType.CLICK, + goog.bind(this.onTldRemove_, this, remBtn)); + this.typeCounts['reg-tlds']++; + tldInputElt.value = ''; +}; + + +/** + * Click handler for TLD remove button. + * @param {!Element} remBtn The remove button. + * @private + */ +registry.registrar.AdminSettings.prototype.onTldRemove_ = + function(remBtn) { + goog.dom.removeNode(goog.dom.getParentElement(remBtn)); +}; diff --git a/java/google/registry/ui/js/registrar/console.js b/java/google/registry/ui/js/registrar/console.js index 9c8d639da..6542c03ea 100644 --- a/java/google/registry/ui/js/registrar/console.js +++ b/java/google/registry/ui/js/registrar/console.js @@ -21,6 +21,7 @@ goog.require('goog.dom.classlist'); goog.require('goog.net.XhrIo'); goog.require('registry.Console'); goog.require('registry.Resource'); +goog.require('registry.registrar.AdminSettings'); goog.require('registry.registrar.Contact'); goog.require('registry.registrar.ContactSettings'); goog.require('registry.registrar.ContactUs'); @@ -76,20 +77,43 @@ registry.registrar.Console = function(params) { this.lastActiveNavElt; /** + * A map from the URL fragment to the component to show. + * * @type {!Object.} */ this.pageMap = {}; + // Homepage. Displayed when there's no fragment, or when the fragment doesn't + // correspond to any view + this.pageMap[''] = registry.registrar.Dashboard; + // Updating the Registrar settings this.pageMap['security-settings'] = registry.registrar.SecuritySettings; this.pageMap['contact-settings'] = registry.registrar.ContactSettings; this.pageMap['whois-settings'] = registry.registrar.WhoisSettings; this.pageMap['contact-us'] = registry.registrar.ContactUs; this.pageMap['resources'] = registry.registrar.Resources; + // For admin use. The relevant tab is only shown in Console.soy for admins, + // but we also need to remove it here, otherwise it'd still be accessible if + // the user manually puts '#admin-settings' in the URL. + // + // Both the Console.soy and here, the "hiding the admin console for non + // admins" is purely for "aesthetic / design" reasons and have NO security + // implications. + // + // The security implications are only in the backend where we make sure all + // changes are made by users with the correct access (in other words - we + // don't trust the client-side to secure our application anyway) + if (this.params.isAdmin) { + this.pageMap['admin-settings'] = registry.registrar.AdminSettings; + } + + // sending EPPs through the console. Currently hidden (doesn't have a "tab") + // but still accessible if the user manually puts #domain (or other) in the + // fragment this.pageMap['contact'] = registry.registrar.Contact; this.pageMap['domain'] = registry.registrar.Domain; this.pageMap['host'] = registry.registrar.Host; - this.pageMap[''] = registry.registrar.Dashboard; }; goog.inherits(registry.registrar.Console, registry.Console); diff --git a/java/google/registry/ui/js/registrar/contact_settings.js b/java/google/registry/ui/js/registrar/contact_settings.js index d690bc79a..5284f6ad1 100644 --- a/java/google/registry/ui/js/registrar/contact_settings.js +++ b/java/google/registry/ui/js/registrar/contact_settings.js @@ -45,7 +45,7 @@ goog.forwardDeclare('registry.registrar.Console'); registry.registrar.ContactSettings = function(console, resource) { registry.registrar.ContactSettings.base( this, 'constructor', console, resource, - registry.soy.registrar.contacts.contact, null); + registry.soy.registrar.contacts.contact, console.params.isOwner, null); }; goog.inherits(registry.registrar.ContactSettings, registry.ResourceComponent); diff --git a/java/google/registry/ui/js/registrar/contact_us.js b/java/google/registry/ui/js/registrar/contact_us.js index 0d19c09ad..73b1f97eb 100644 --- a/java/google/registry/ui/js/registrar/contact_us.js +++ b/java/google/registry/ui/js/registrar/contact_us.js @@ -34,7 +34,7 @@ goog.forwardDeclare('registry.registrar.Console'); registry.registrar.ContactUs = function(console, resource) { registry.registrar.ContactUs.base( this, 'constructor', console, resource, - registry.soy.registrar.console.contactUs, null); + registry.soy.registrar.console.contactUs, console.params.isOwner, null); }; goog.inherits(registry.registrar.ContactUs, registry.ResourceComponent); diff --git a/java/google/registry/ui/js/registrar/dashboard.js b/java/google/registry/ui/js/registrar/dashboard.js index 1fdb7b980..17ffbaad7 100644 --- a/java/google/registry/ui/js/registrar/dashboard.js +++ b/java/google/registry/ui/js/registrar/dashboard.js @@ -52,7 +52,7 @@ goog.inherits(registry.registrar.Dashboard, registry.Component); /** @override */ registry.registrar.Dashboard.prototype.bindToDom = function(id) { registry.registrar.Dashboard.base(this, 'bindToDom', ''); - goog.dom.removeChildren(goog.dom.getRequiredElement('reg-appbar')); + goog.dom.removeChildren(goog.dom.getRequiredElement('reg-app-buttons')); goog.soy.renderElement(goog.dom.getElement('reg-content'), registry.soy.registrar.console.dashboard, this.console.params); diff --git a/java/google/registry/ui/js/registrar/epp_session.js b/java/google/registry/ui/js/registrar/epp_session.js index 188eae24d..ab0c9e89e 100644 --- a/java/google/registry/ui/js/registrar/epp_session.js +++ b/java/google/registry/ui/js/registrar/epp_session.js @@ -33,7 +33,9 @@ goog.forwardDeclare('registry.registrar.Console'); */ registry.registrar.EppSession = function(console) { registry.registrar.EppSession.base( - this, 'constructor', new goog.Uri('/registrar-xhr'), + this, 'constructor', + new goog.Uri('/registrar-xhr') + .setParameterValue('clientId', console.params.clientId), console.params.xsrfToken, registry.Session.ContentType.EPP); diff --git a/java/google/registry/ui/js/registrar/main.js b/java/google/registry/ui/js/registrar/main.js index dfc5817e9..1a8ad1df4 100644 --- a/java/google/registry/ui/js/registrar/main.js +++ b/java/google/registry/ui/js/registrar/main.js @@ -28,6 +28,8 @@ goog.require('registry.registrar.Console'); * * @param {string} xsrfToken populated by server-side soy template. * @param {string} clientId The registrar clientId. + * @param {boolean} isAdmin + * @param {boolean} isOwner * @param {string} productName the product name displayed by the UI. * @param {string} integrationEmail * @param {string} supportEmail @@ -36,13 +38,15 @@ goog.require('registry.registrar.Console'); * @param {string} technicalDocsUrl * @export */ -registry.registrar.main = function(xsrfToken, clientId, productName, - integrationEmail, supportEmail, +registry.registrar.main = function(xsrfToken, clientId, isAdmin, isOwner, + productName, integrationEmail, supportEmail, announcementsEmail, supportPhoneNumber, technicalDocsUrl) { new registry.registrar.Console({ xsrfToken: xsrfToken, clientId: clientId, + isAdmin: isAdmin, + isOwner: isOwner, productName: productName, integrationEmail: integrationEmail, supportEmail: supportEmail, diff --git a/java/google/registry/ui/js/registrar/resources.js b/java/google/registry/ui/js/registrar/resources.js index 5f188a38d..e6116813e 100644 --- a/java/google/registry/ui/js/registrar/resources.js +++ b/java/google/registry/ui/js/registrar/resources.js @@ -34,7 +34,7 @@ goog.forwardDeclare('registry.registrar.Console'); registry.registrar.Resources = function(console, resource) { registry.registrar.Resources.base( this, 'constructor', console, resource, - registry.soy.registrar.console.resources, null); + registry.soy.registrar.console.resources, console.params.isOwner, null); }; goog.inherits(registry.registrar.Resources, registry.ResourceComponent); diff --git a/java/google/registry/ui/js/registrar/security_settings.js b/java/google/registry/ui/js/registrar/security_settings.js index 1a3f2ed34..8c092b63e 100644 --- a/java/google/registry/ui/js/registrar/security_settings.js +++ b/java/google/registry/ui/js/registrar/security_settings.js @@ -39,7 +39,7 @@ goog.forwardDeclare('registry.registrar.Console'); registry.registrar.SecuritySettings = function(console, resource) { registry.registrar.SecuritySettings.base( this, 'constructor', console, resource, - registry.soy.registrar.security.settings, null); + registry.soy.registrar.security.settings, console.params.isOwner, null); }; goog.inherits(registry.registrar.SecuritySettings, registry.ResourceComponent); @@ -103,5 +103,4 @@ registry.registrar.SecuritySettings.prototype.onIpAdd_ = function() { registry.registrar.SecuritySettings.prototype.onIpRemove_ = function(remBtn) { goog.dom.removeNode(goog.dom.getParentElement(remBtn)); - this.typeCounts['reg-ips']--; }; diff --git a/java/google/registry/ui/js/registrar/whois_settings.js b/java/google/registry/ui/js/registrar/whois_settings.js index 68297dbb7..a87d0dca6 100644 --- a/java/google/registry/ui/js/registrar/whois_settings.js +++ b/java/google/registry/ui/js/registrar/whois_settings.js @@ -34,7 +34,7 @@ goog.forwardDeclare('registry.registrar.Console'); registry.registrar.WhoisSettings = function(console, resource) { registry.registrar.WhoisSettings.base( this, 'constructor', console, resource, - registry.soy.registrar.whois.settings, null); + registry.soy.registrar.whois.settings, console.params.isOwner, null); }; goog.inherits(registry.registrar.WhoisSettings, registry.ResourceComponent); diff --git a/java/google/registry/ui/js/registrar/xml_resource_component.js b/java/google/registry/ui/js/registrar/xml_resource_component.js index b3b9a083d..2f736bc08 100644 --- a/java/google/registry/ui/js/registrar/xml_resource_component.js +++ b/java/google/registry/ui/js/registrar/xml_resource_component.js @@ -37,7 +37,7 @@ goog.forwardDeclare('registry.registrar.Console'); registry.registrar.XmlResourceComponent = function( itemTmpl, eppTmpls, console) { registry.registrar.XmlResourceComponent.base( - this, 'constructor', console, itemTmpl); + this, 'constructor', console, itemTmpl, console.params.isOwner); /** @type {!Object} */ this.eppTmpls = eppTmpls; diff --git a/java/google/registry/ui/js/resource_component.js b/java/google/registry/ui/js/resource_component.js index 5f1084b56..469260978 100644 --- a/java/google/registry/ui/js/resource_component.js +++ b/java/google/registry/ui/js/resource_component.js @@ -33,6 +33,7 @@ goog.forwardDeclare('registry.Resource'); * @param {!registry.Console} console console singleton. * @param {!registry.Resource} resource the RESTful resource. * @param {!Function} itemTmpl + * @param {boolean} isEditable if true, the "edit" button will be enabled * @param {Function} renderSetCb may be null if this resource is only * ever an item. * @constructor @@ -42,8 +43,10 @@ registry.ResourceComponent = function( console, resource, itemTmpl, + isEditable, renderSetCb) { - registry.ResourceComponent.base(this, 'constructor', console, itemTmpl); + registry.ResourceComponent.base( + this, 'constructor', console, itemTmpl, isEditable); /** @type {!registry.Resource} */ this.resource = resource; @@ -108,7 +111,7 @@ registry.ResourceComponent.prototype.handleFetchItem = function(id, rsp) { } else if ('set' in rsp && this.renderSetCb != null) { // XXX: This conditional logic should be hoisted to edit_item when // collection support is improved. - goog.dom.removeChildren(goog.dom.getRequiredElement('reg-appbar')); + goog.dom.removeChildren(goog.dom.getRequiredElement('reg-app-buttons')); this.renderSetCb(goog.dom.getRequiredElement('reg-content'), rsp); } else { registry.util.log('unknown message type in handleFetchItem'); diff --git a/java/google/registry/ui/server/BUILD b/java/google/registry/ui/server/BUILD index af1bc6881..e05f76536 100644 --- a/java/google/registry/ui/server/BUILD +++ b/java/google/registry/ui/server/BUILD @@ -13,12 +13,16 @@ java_library( "//java/google/registry/ui/css:registrar_dbg.css.js", ], deps = [ + "//java/google/registry/config", "//java/google/registry/model", "//java/google/registry/ui", "//java/google/registry/ui/forms", "//java/google/registry/util", "@com_google_appengine_api_1_0_sdk", "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_flogger", + "@com_google_flogger_system_backend", "@com_google_guava", "@com_google_re2j", "@io_bazel_rules_closure//closure/templates", diff --git a/java/google/registry/ui/server/registrar/SendEmailUtils.java b/java/google/registry/ui/server/SendEmailUtils.java similarity index 65% rename from java/google/registry/ui/server/registrar/SendEmailUtils.java rename to java/google/registry/ui/server/SendEmailUtils.java index 90fcc5469..278ab0f4f 100644 --- a/java/google/registry/ui/server/registrar/SendEmailUtils.java +++ b/java/google/registry/ui/server/SendEmailUtils.java @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package google.registry.ui.server.registrar; +package google.registry.ui.server; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.toArray; import com.google.common.base.Joiner; -import com.google.common.collect.Streams; +import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; import google.registry.util.SendEmailService; @@ -37,30 +37,40 @@ public class SendEmailUtils { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private final String gSuiteOutgoingEmailAddress; - private final String gSuiteOutoingEmailDisplayName; + private final String gSuiteOutgoingEmailDisplayName; private final SendEmailService emailService; + private final ImmutableList registrarChangesNotificationEmailAddresses; @Inject public SendEmailUtils( @Config("gSuiteOutgoingEmailAddress") String gSuiteOutgoingEmailAddress, - @Config("gSuiteOutoingEmailDisplayName") String gSuiteOutoingEmailDisplayName, + @Config("gSuiteOutgoingEmailDisplayName") String gSuiteOutgoingEmailDisplayName, + @Config("registrarChangesNotificationEmailAddresses") + ImmutableList registrarChangesNotificationEmailAddresses, SendEmailService emailService) { this.gSuiteOutgoingEmailAddress = gSuiteOutgoingEmailAddress; - this.gSuiteOutoingEmailDisplayName = gSuiteOutoingEmailDisplayName; + this.gSuiteOutgoingEmailDisplayName = gSuiteOutgoingEmailDisplayName; this.emailService = emailService; + this.registrarChangesNotificationEmailAddresses = registrarChangesNotificationEmailAddresses; } /** - * Sends an email from Nomulus to the specified recipient(s). Returns true iff sending was - * successful. + * Sends an email from Nomulus to the registrarChangesNotificationEmailAddresses. Returns true iff + * sending to at least 1 address was successful. + * + *

This means that if there are no recepients ({@link #hasRecepients} returns false), this will + * return false even thought no error happened. + * + *

This also means that if there are multiple recepients, it will return true even if some (but + * not all) of the recepients had an error. */ - public boolean sendEmail(Iterable addresses, final String subject, String body) { + public boolean sendEmail(final String subject, String body) { try { Message msg = emailService.createMessage(); msg.setFrom( - new InternetAddress(gSuiteOutgoingEmailAddress, gSuiteOutoingEmailDisplayName)); + new InternetAddress(gSuiteOutgoingEmailAddress, gSuiteOutgoingEmailDisplayName)); List emails = - Streams.stream(addresses) + registrarChangesNotificationEmailAddresses.stream() .map( emailAddress -> { try { @@ -85,9 +95,17 @@ public class SendEmailUtils { } catch (Throwable t) { logger.atSevere().withCause(t).log( "Could not email to addresses %s with subject '%s'.", - Joiner.on(", ").join(addresses), subject); + Joiner.on(", ").join(registrarChangesNotificationEmailAddresses), subject); return false; } return true; } + + /** + * Returns whether there are any recepients set up. {@link #sendEmail} will always return false if + * there are no recepients. + */ + public boolean hasRecepients() { + return !registrarChangesNotificationEmailAddresses.isEmpty(); + } } diff --git a/java/google/registry/ui/server/otesetup/BUILD b/java/google/registry/ui/server/otesetup/BUILD new file mode 100644 index 000000000..4ce526598 --- /dev/null +++ b/java/google/registry/ui/server/otesetup/BUILD @@ -0,0 +1,43 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +java_library( + name = "otesetup", + srcs = glob(["*.java"]), + resources = [ + "//java/google/registry/ui/css:registrar_bin.css.js", + "//java/google/registry/ui/css:registrar_dbg.css.js", + ], + deps = [ + "//java/google/registry/config", + "//java/google/registry/export/sheet", + "//java/google/registry/flows", + "//java/google/registry/model", + "//java/google/registry/request", + "//java/google/registry/request/auth", + "//java/google/registry/security", + "//java/google/registry/ui/forms", + "//java/google/registry/ui/server", + "//java/google/registry/ui/soy:soy_java_wrappers", + "//java/google/registry/ui/soy/otesetup:soy_java_wrappers", + "//java/google/registry/util", + "//third_party/objectify:objectify-v4_1", + "@com_google_appengine_api_1_0_sdk", + "@com_google_auto_value", + "@com_google_code_findbugs_jsr305", + "@com_google_dagger", + "@com_google_flogger", + "@com_google_flogger_system_backend", + "@com_google_guava", + "@com_google_monitoring_client_metrics", + "@com_google_re2j", + "@io_bazel_rules_closure//closure/templates", + "@javax_inject", + "@javax_servlet_api", + "@joda_time", + "@org_joda_money", + ], +) diff --git a/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java b/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java new file mode 100644 index 000000000..6af97abf8 --- /dev/null +++ b/java/google/registry/ui/server/otesetup/ConsoleOteSetupAction.java @@ -0,0 +1,244 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.otesetup; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.net.HttpHeaders.LOCATION; +import static com.google.common.net.HttpHeaders.X_FRAME_OPTIONS; +import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; +import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY; + +import com.google.appengine.api.users.User; +import com.google.appengine.api.users.UserService; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.common.flogger.FluentLogger; +import com.google.common.io.Resources; +import com.google.common.net.MediaType; +import com.google.template.soy.shared.SoyCssRenamingMap; +import com.google.template.soy.tofu.SoyTofu; +import google.registry.config.RegistryConfig.Config; +import google.registry.config.RegistryEnvironment; +import google.registry.model.OteAccountBuilder; +import google.registry.request.Action; +import google.registry.request.Action.Method; +import google.registry.request.Parameter; +import google.registry.request.RequestMethod; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.request.auth.AuthResult; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.security.XsrfTokenManager; +import google.registry.ui.server.SendEmailUtils; +import google.registry.ui.server.SoyTemplateUtils; +import google.registry.ui.soy.otesetup.ConsoleSoyInfo; +import google.registry.util.StringGenerator; +import java.util.HashMap; +import java.util.Optional; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; + +/** + * Action that serves OT&E setup web page. + * + *

This Action does 2 things: - for GET, just returns the form that asks for the clientId and + * email. - for POST, receives the clientId and email and generates the OTE entities. + * + *

TODO(b/120201577): once we can have 2 different Actions with the same path (different + * Methods), separate this class to 2 Actions. + */ +@Action( + path = ConsoleOteSetupAction.PATH, + method = {Method.POST, Method.GET}, + auth = Auth.AUTH_PUBLIC) +public final class ConsoleOteSetupAction implements Runnable { + + private static final int PASSWORD_LENGTH = 16; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + public static final String PATH = "/registrar-ote-setup"; + + private static final Supplier TOFU_SUPPLIER = + SoyTemplateUtils.createTofuSupplier( + google.registry.ui.soy.ConsoleSoyInfo.getInstance(), + google.registry.ui.soy.FormsSoyInfo.getInstance(), + google.registry.ui.soy.otesetup.ConsoleSoyInfo.getInstance()); + + @VisibleForTesting // webdriver and screenshot tests need this + public static final Supplier CSS_RENAMING_MAP_SUPPLIER = + SoyTemplateUtils.createCssRenamingMapSupplier( + Resources.getResource("google/registry/ui/css/registrar_bin.css.js"), + Resources.getResource("google/registry/ui/css/registrar_dbg.css.js")); + + @Inject HttpServletRequest req; + @Inject @RequestMethod Method method; + @Inject Response response; + @Inject AuthenticatedRegistrarAccessor registrarAccessor; + @Inject UserService userService; + @Inject XsrfTokenManager xsrfTokenManager; + @Inject AuthResult authResult; + @Inject RegistryEnvironment registryEnvironment; + @Inject SendEmailUtils sendEmailUtils; + @Inject @Config("logoFilename") String logoFilename; + @Inject @Config("productName") String productName; + @Inject @Config("base64StringGenerator") StringGenerator passwordGenerator; + @Inject @Parameter("clientId") Optional clientId; + @Inject @Parameter("email") Optional email; + @Inject @Parameter("password") Optional optionalPassword; + + @Inject ConsoleOteSetupAction() {} + + @Override + public void run() { + response.setHeader(X_FRAME_OPTIONS, "SAMEORIGIN"); // Disallow iframing. + response.setHeader("X-Ui-Compatible", "IE=edge"); // Ask IE not to be silly. + + logger.atInfo().log( + "User %s is accessing the OT&E setup page. Method= %s", + registrarAccessor.userIdForLogging(), method); + checkState(registryEnvironment != RegistryEnvironment.PRODUCTION, "Can't create OT&E in prod"); + if (!authResult.userAuthInfo().isPresent()) { + response.setStatus(SC_MOVED_TEMPORARILY); + String location; + try { + location = userService.createLoginURL(req.getRequestURI()); + } catch (IllegalArgumentException e) { + // UserServiceImpl.createLoginURL() throws IllegalArgumentException if underlying API call + // returns an error code of NOT_ALLOWED. createLoginURL() assumes that the error is caused + // by an invalid URL. But in fact, the error can also occur if UserService doesn't have any + // user information, which happens when the request has been authenticated as internal. In + // this case, we want to avoid dying before we can send the redirect, so just redirect to + // the root path. + location = "/"; + } + response.setHeader(LOCATION, location); + return; + } + User user = authResult.userAuthInfo().get().user(); + + // Using HashMap to allow null values + HashMap data = new HashMap<>(); + data.put("logoFilename", logoFilename); + data.put("productName", productName); + data.put("username", user.getNickname()); + data.put("logoutUrl", userService.createLogoutURL(PATH)); + data.put("xsrfToken", xsrfTokenManager.generateToken(user.getEmail())); + response.setContentType(MediaType.HTML_UTF_8); + + if (!registrarAccessor.isAdmin()) { + response.setStatus(SC_FORBIDDEN); + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.WHOAREYOU) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + return; + } + switch (method) { + case POST: + runPost(data); + return; + case GET: + runGet(data); + return; + default: + return; + } + } + + private void runPost(HashMap data) { + // This is intentionally outside of the "try/catch", since not having these fields means someone + // tried to "manually" POST to this page. No need to send out a "pretty" result in that case. + checkState(clientId.isPresent() && email.isPresent(), "Must supply clientId and email"); + + data.put("baseClientId", clientId.get()); + data.put("contactEmail", email.get()); + + try { + String password = optionalPassword.orElse(passwordGenerator.createString(PASSWORD_LENGTH)); + ImmutableMap clientIdToTld = + OteAccountBuilder.forClientId(clientId.get()) + .addContact(email.get()) + .setPassword(password) + .buildAndPersist(); + + sendExternalUpdates(clientIdToTld); + + data.put("clientIdToTld", clientIdToTld); + data.put("password", password); + + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.RESULT_SUCCESS) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + } catch (Throwable e) { + logger.atWarning().withCause(e).log( + "Failed to setup OT&E. clientId: %s, email: %s", clientId.get(), email.get()); + data.put("errorMessage", e.getMessage()); + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.FORM_PAGE) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + } + } + + private void runGet(HashMap data) { + // set the values to pre-fill, if given + data.put("baseClientId", clientId.orElse(null)); + data.put("contactEmail", email.orElse(null)); + + response.setPayload( + TOFU_SUPPLIER + .get() + .newRenderer(ConsoleSoyInfo.FORM_PAGE) + .setCssRenamingMap(CSS_RENAMING_MAP_SUPPLIER.get()) + .setData(data) + .render()); + } + + + private void sendExternalUpdates(ImmutableMap clientIdToTld) { + if (!sendEmailUtils.hasRecepients()) { + return; + } + String environment = Ascii.toLowerCase(String.valueOf(registryEnvironment)); + StringBuilder builder = new StringBuilder(); + builder.append( + String.format( + "The following entities were created in %s by %s:\n", + environment, registrarAccessor.userIdForLogging())); + clientIdToTld.forEach( + (clientId, tld) -> + builder.append( + String.format(" Registrar %s with access to TLD %s\n", clientId, tld))); + builder.append(String.format("Gave user %s web access to these Registrars\n", email.get())); + sendEmailUtils.sendEmail( + String.format( + "OT&E for registrar %s created in %s", + clientId.get(), + environment), + builder.toString()); + } +} diff --git a/java/google/registry/ui/server/registrar/AuthenticatedRegistrarAccessor.java b/java/google/registry/ui/server/registrar/AuthenticatedRegistrarAccessor.java deleted file mode 100644 index f1a531d93..000000000 --- a/java/google/registry/ui/server/registrar/AuthenticatedRegistrarAccessor.java +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2018 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.ui.server.registrar; - -import static google.registry.model.ofy.ObjectifyService.ofy; - -import com.google.appengine.api.users.User; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableSetMultimap; -import com.google.common.flogger.FluentLogger; -import google.registry.config.RegistryConfig.Config; -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import google.registry.request.HttpException.ForbiddenException; -import google.registry.request.auth.AuthResult; -import google.registry.request.auth.UserAuthInfo; -import javax.annotation.concurrent.Immutable; -import javax.inject.Inject; - -/** - * Allows access only to {@link Registrar}s the current user has access to. - * - *

A user has OWNER role on a Registrar if there exists a {@link RegistrarContact} with - * that user's gaeId and the registrar as a parent. - * - *

An admin has in addition OWNER role on {@link #registryAdminClientId}. - * - *

An admin also has ADMIN role on ALL registrars. - */ -@Immutable -public class AuthenticatedRegistrarAccessor { - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - - /** The role under which access is granted. */ - public enum Role { - OWNER, - ADMIN - } - - AuthResult authResult; - String registryAdminClientId; - - /** - * Gives all roles a user has for a given clientId. - * - *

The order is significant, with "more specific to this user" coming first. - */ - private final ImmutableSetMultimap roleMap; - - @Inject - public AuthenticatedRegistrarAccessor( - AuthResult authResult, @Config("registryAdminClientId") String registryAdminClientId) { - this.authResult = authResult; - this.registryAdminClientId = registryAdminClientId; - this.roleMap = createRoleMap(authResult, registryAdminClientId); - - logger.atInfo().log( - "%s has the following roles: %s", authResult.userIdForLogging(), roleMap); - } - - /** - * A map that gives all roles a user has for a given clientId. - * - *

Throws a {@link ForbiddenException} if the user is not logged in. - * - *

The result is ordered starting from "most specific to this user". - * - *

If you want to load the {@link Registrar} object from these (or any other) {@code clientId}, - * in order to perform actions on behalf of a user, you must use {@link #getRegistrar} which makes - * sure the user has permissions. - * - *

Note that this is an OPTIONAL step in the authentication - only used if we don't have any - * other clue as to the requested {@code clientId}. It is perfectly OK to get a {@code clientId} - * from any other source, as long as the registrar is then loaded using {@link #getRegistrar}. - */ - public ImmutableSetMultimap getAllClientIdWithRoles() { - return roleMap; - } - - /** - * "Guesses" which client ID the user wants from all those they have access to. - * - *

If no such ClientIds exist, throws a ForbiddenException. - * - *

This should be the ClientId "most likely wanted by the user". - * - *

If you want to load the {@link Registrar} object from this (or any other) {@code clientId}, - * in order to perform actions on behalf of a user, you must use {@link #getRegistrar} which makes - * sure the user has permissions. - * - *

Note that this is an OPTIONAL step in the authentication - only used if we don't have any - * other clue as to the requested {@code clientId}. It is perfectly OK to get a {@code clientId} - * from any other source, as long as the registrar is then loaded using {@link #getRegistrar}. - */ - public String guessClientId() { - verifyLoggedIn(); - return getAllClientIdWithRoles().keySet().stream() - .findFirst() - .orElseThrow( - () -> - new ForbiddenException( - String.format( - "%s isn't associated with any registrar", - authResult.userIdForLogging()))); - } - - /** - * Loads a Registrar IFF the user is authorized. - * - *

Throws a {@link ForbiddenException} if the user is not logged in, or not authorized to - * access the requested registrar. - * - * @param clientId ID of the registrar we request - */ - public Registrar getRegistrar(String clientId) { - verifyLoggedIn(); - - ImmutableSet roles = getAllClientIdWithRoles().get(clientId); - - if (roles.isEmpty()) { - throw new ForbiddenException( - String.format( - "%s doesn't have access to registrar %s", - authResult.userIdForLogging(), clientId)); - } - - Registrar registrar = - Registrar.loadByClientId(clientId) - .orElseThrow( - () -> new ForbiddenException(String.format("Registrar %s not found", clientId))); - - if (!clientId.equals(registrar.getClientId())) { - logger.atSevere().log( - "registrarLoader.apply(clientId) returned a Registrar with a different clientId. " - + "Requested: %s, returned: %s.", - clientId, registrar.getClientId()); - throw new ForbiddenException("Internal error - please check logs"); - } - - logger.atInfo().log( - "%s has %s access to registrar %s.", authResult.userIdForLogging(), roles, clientId); - return registrar; - } - - private static ImmutableSetMultimap createRoleMap( - AuthResult authResult, String registryAdminClientId) { - - if (!authResult.userAuthInfo().isPresent()) { - return ImmutableSetMultimap.of(); - } - - UserAuthInfo userAuthInfo = authResult.userAuthInfo().get(); - - boolean isAdmin = userAuthInfo.isUserAdmin(); - User user = userAuthInfo.user(); - - ImmutableSetMultimap.Builder builder = new ImmutableSetMultimap.Builder<>(); - - ofy() - .load() - .type(RegistrarContact.class) - .filter("gaeUserId", user.getUserId()) - .forEach( - contact -> - builder - .put(contact.getParent().getName(), Role.OWNER)); - if (isAdmin && !Strings.isNullOrEmpty(registryAdminClientId)) { - builder - .put(registryAdminClientId, Role.OWNER); - } - - if (isAdmin) { - // Admins have access to all registrars - ofy() - .load() - .type(Registrar.class) - .forEach(registrar -> builder.put(registrar.getClientId(), Role.ADMIN)); - } - - return builder.build(); - } - - private void verifyLoggedIn() { - if (!authResult.userAuthInfo().isPresent()) { - throw new ForbiddenException("Not logged in"); - } - } -} diff --git a/java/google/registry/ui/server/registrar/ConsoleUiAction.java b/java/google/registry/ui/server/registrar/ConsoleUiAction.java index be45d805a..e6c3331a2 100644 --- a/java/google/registry/ui/server/registrar/ConsoleUiAction.java +++ b/java/google/registry/ui/server/registrar/ConsoleUiAction.java @@ -16,6 +16,8 @@ package google.registry.ui.server.registrar; import static com.google.common.net.HttpHeaders.LOCATION; import static com.google.common.net.HttpHeaders.X_FRAME_OPTIONS; +import static google.registry.request.auth.AuthenticatedRegistrarAccessor.Role.ADMIN; +import static google.registry.request.auth.AuthenticatedRegistrarAccessor.Role.OWNER; import static google.registry.ui.server.registrar.RegistrarConsoleModule.PARAM_CLIENT_ID; import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; import static javax.servlet.http.HttpServletResponse.SC_MOVED_TEMPORARILY; @@ -35,14 +37,15 @@ import com.google.template.soy.tofu.SoyTofu; import google.registry.config.RegistryConfig.Config; import google.registry.model.registrar.Registrar; import google.registry.request.Action; -import google.registry.request.HttpException.ForbiddenException; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import google.registry.request.auth.AuthResult; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.RegistrarAccessDeniedException; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.Role; import google.registry.security.XsrfTokenManager; import google.registry.ui.server.SoyTemplateUtils; -import google.registry.ui.server.registrar.AuthenticatedRegistrarAccessor.Role; import google.registry.ui.soy.registrar.ConsoleSoyInfo; import java.util.Optional; import javax.inject.Inject; @@ -139,6 +142,8 @@ public final class ConsoleUiAction implements Runnable { try { clientId = paramClientId.orElse(registrarAccessor.guessClientId()); data.put("clientId", clientId); + data.put("isOwner", roleMap.containsEntry(clientId, OWNER)); + data.put("isAdmin", roleMap.containsEntry(clientId, ADMIN)); // We want to load the registrar even if we won't use it later (even if we remove the // requireFeeExtension) - to make sure the user indeed has access to the guessed registrar. @@ -149,7 +154,7 @@ public final class ConsoleUiAction implements Runnable { // because the requests come from the browser, and can easily be faked) Registrar registrar = registrarAccessor.getRegistrar(clientId); data.put("requireFeeExtension", registrar.getPremiumPriceAckRequired()); - } catch (ForbiddenException e) { + } catch (RegistrarAccessDeniedException e) { logger.atWarning().withCause(e).log( "User %s doesn't have access to registrar console.", authResult.userIdForLogging()); response.setStatus(SC_FORBIDDEN); diff --git a/java/google/registry/ui/server/registrar/OteStatusAction.java b/java/google/registry/ui/server/registrar/OteStatusAction.java new file mode 100644 index 000000000..fcd46cd56 --- /dev/null +++ b/java/google/registry/ui/server/registrar/OteStatusAction.java @@ -0,0 +1,121 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.ui.server.registrar; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.security.JsonResponseHelper.Status.ERROR; +import static google.registry.security.JsonResponseHelper.Status.SUCCESS; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.common.flogger.FluentLogger; +import google.registry.model.OteAccountBuilder; +import google.registry.model.OteStats; +import google.registry.model.OteStats.StatType; +import google.registry.model.registrar.Registrar; +import google.registry.model.registrar.Registrar.Type; +import google.registry.request.Action; +import google.registry.request.JsonActionRunner; +import google.registry.request.auth.Auth; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.RegistrarAccessDeniedException; +import google.registry.security.JsonResponseHelper; +import java.util.Map; +import java.util.Optional; +import javax.inject.Inject; + +/** + * Admin servlet that allows creating or updating a registrar. Deletes are not allowed so as to + * preserve history. + */ +@Action(path = OteStatusAction.PATH, method = Action.Method.POST, auth = Auth.AUTH_PUBLIC_LOGGED_IN) +public final class OteStatusAction implements Runnable, JsonActionRunner.JsonAction { + + public static final String PATH = "/registrar-ote-status"; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final String CLIENT_ID_PARAM = "clientId"; + private static final String COMPLETED_PARAM = "completed"; + private static final String DETAILS_PARAM = "details"; + private static final String STAT_TYPE_DESCRIPTION_PARAM = "description"; + private static final String STAT_TYPE_REQUIREMENT_PARAM = "requirement"; + private static final String STAT_TYPE_TIMES_PERFORMED_PARAM = "timesPerformed"; + + @Inject AuthenticatedRegistrarAccessor registrarAccessor; + @Inject JsonActionRunner jsonActionRunner; + + @Inject + OteStatusAction() {} + + @Override + public void run() { + jsonActionRunner.run(this); + } + + @Override + public Map handleJsonRequest(Map input) { + try { + checkArgument(input != null, "Malformed JSON"); + + String oteClientId = (String) input.get(CLIENT_ID_PARAM); + checkArgument( + !Strings.isNullOrEmpty(oteClientId), "Missing key for OT&E client: %s", CLIENT_ID_PARAM); + + String baseClientId = OteAccountBuilder.getBaseClientId(oteClientId); + Registrar oteRegistrar = registrarAccessor.getRegistrar(oteClientId); + verifyOteAccess(baseClientId); + checkArgument( + Type.OTE.equals(oteRegistrar.getType()), + "Registrar with ID %s is not an OT&E registrar", + oteClientId); + + OteStats oteStats = OteStats.getFromRegistrar(baseClientId); + return JsonResponseHelper.create( + SUCCESS, "OT&E check completed successfully", convertOteStats(baseClientId, oteStats)); + } catch (Throwable e) { + logger.atWarning().withCause(e).log( + "Failed to verify OT&E status for registrar with input %s", input); + return JsonResponseHelper.create( + ERROR, Optional.ofNullable(e.getMessage()).orElse("Unspecified error")); + } + } + + private void verifyOteAccess(String baseClientId) throws RegistrarAccessDeniedException { + for (String oteClientId : OteAccountBuilder.createClientIdToTldMap(baseClientId).keySet()) { + registrarAccessor.verifyAccess(oteClientId); + } + } + + private Map convertOteStats(String baseClientId, OteStats oteStats) { + return ImmutableMap.of( + CLIENT_ID_PARAM, baseClientId, + COMPLETED_PARAM, oteStats.getFailures().isEmpty(), + DETAILS_PARAM, + StatType.REQUIRED_STAT_TYPES.stream() + .map(statType -> convertSingleRequirement(statType, oteStats.getCount(statType))) + .collect(toImmutableList())); + } + + private Map convertSingleRequirement(StatType statType, int count) { + int requirement = statType.getRequirement(); + return ImmutableMap.of( + STAT_TYPE_DESCRIPTION_PARAM, statType.getDescription(), + STAT_TYPE_REQUIREMENT_PARAM, requirement, + STAT_TYPE_TIMES_PERFORMED_PARAM, count, + COMPLETED_PARAM, count >= requirement); + } +} diff --git a/java/google/registry/ui/server/registrar/RegistrarConsoleMetrics.java b/java/google/registry/ui/server/registrar/RegistrarConsoleMetrics.java index 338a595e4..e1993bb1c 100644 --- a/java/google/registry/ui/server/registrar/RegistrarConsoleMetrics.java +++ b/java/google/registry/ui/server/registrar/RegistrarConsoleMetrics.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableSet; import com.google.monitoring.metrics.IncrementableMetric; import com.google.monitoring.metrics.LabelDescriptor; import com.google.monitoring.metrics.MetricRegistryImpl; -import google.registry.ui.server.registrar.AuthenticatedRegistrarAccessor.Role; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.Role; import javax.inject.Inject; final class RegistrarConsoleMetrics { diff --git a/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java b/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java index 9026c5ce5..f4b7d55b1 100644 --- a/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java +++ b/java/google/registry/ui/server/registrar/RegistrarConsoleModule.java @@ -16,6 +16,7 @@ package google.registry.ui.server.registrar; import static google.registry.request.RequestParameters.extractOptionalParameter; +import static google.registry.request.RequestParameters.extractRequiredParameter; import dagger.Module; import dagger.Provides; @@ -31,7 +32,31 @@ public final class RegistrarConsoleModule { @Provides @Parameter(PARAM_CLIENT_ID) - static Optional provideClientId(HttpServletRequest req) { + static Optional provideOptionalClientId(HttpServletRequest req) { return extractOptionalParameter(req, PARAM_CLIENT_ID); } + + @Provides + @Parameter(PARAM_CLIENT_ID) + static String provideClientId(HttpServletRequest req) { + return extractRequiredParameter(req, PARAM_CLIENT_ID); + } + + @Provides + @Parameter("email") + static Optional provideOptionalEmail(HttpServletRequest req) { + return extractOptionalParameter(req, "email"); + } + + @Provides + @Parameter("email") + static String provideEmail(HttpServletRequest req) { + return extractRequiredParameter(req, "email"); + } + + @Provides + @Parameter("password") + static Optional provideOptionalPassword(HttpServletRequest req) { + return extractOptionalParameter(req, "password"); + } } diff --git a/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java b/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java index 412d03fa1..1d3f9eaa4 100644 --- a/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java +++ b/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java @@ -14,6 +14,7 @@ package google.registry.ui.server.registrar; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.Sets.difference; import static google.registry.export.sheet.SyncRegistrarsSheetAction.enqueueRegistrarSheetSync; @@ -29,9 +30,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; -import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarContact; @@ -42,11 +43,14 @@ import google.registry.request.HttpException.ForbiddenException; import google.registry.request.JsonActionRunner; import google.registry.request.auth.Auth; import google.registry.request.auth.AuthResult; +import google.registry.request.auth.AuthenticatedRegistrarAccessor; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.RegistrarAccessDeniedException; +import google.registry.request.auth.AuthenticatedRegistrarAccessor.Role; import google.registry.security.JsonResponseHelper; import google.registry.ui.forms.FormException; import google.registry.ui.forms.FormFieldException; import google.registry.ui.server.RegistrarFormFields; -import google.registry.ui.server.registrar.AuthenticatedRegistrarAccessor.Role; +import google.registry.ui.server.SendEmailUtils; import google.registry.util.AppEngineServiceUtils; import google.registry.util.CollectionUtils; import google.registry.util.DiffUtils; @@ -54,6 +58,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; @@ -87,8 +92,6 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA @Inject AuthResult authResult; @Inject RegistryEnvironment registryEnvironment; - @Inject @Config("registrarChangesNotificationEmailAddresses") ImmutableList - registrarChangesNotificationEmailAddresses; @Inject RegistrarSettingsAction() {} private static final Predicate HAS_PHONE = @@ -141,7 +144,7 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA ERROR, Optional.ofNullable(e.getMessage()).orElse("Unspecified error")); } finally { registrarConsoleMetrics.registerSettingsRequest( - clientId, op, registrarAccessor.getAllClientIdWithRoles().get(clientId), status); + clientId, op, registrarAccessor.getRolesForRegistrar(clientId), status); } } @@ -161,7 +164,11 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA } private RegistrarResult read(String clientId) { - return RegistrarResult.create("Success", registrarAccessor.getRegistrar(clientId)); + try { + return RegistrarResult.create("Success", registrarAccessor.getRegistrar(clientId)); + } catch (RegistrarAccessDeniedException e) { + throw new ForbiddenException(e.getMessage(), e); + } } private RegistrarResult update(final Map args, String clientId) { @@ -171,7 +178,12 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA // We load the registrar here rather than outside of the transaction - to make // sure we have the latest version. This one is loaded inside the transaction, so it's // guaranteed to not change before we update it. - Registrar registrar = registrarAccessor.getRegistrar(clientId); + Registrar registrar; + try { + registrar = registrarAccessor.getRegistrar(clientId); + } catch (RegistrarAccessDeniedException e) { + throw new ForbiddenException(e.getMessage(), e); + } // Verify that the registrar hasn't been changed. // To do that - we find the latest update time (or null if the registrar has been // deleted) and compare to the update time from the args. The update time in the args @@ -193,29 +205,31 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA // removed, email the changes to the contacts ImmutableSet contacts = registrar.getContacts(); - // Update the registrar from the request. - Registrar.Builder builder = registrar.asBuilder(); - Set roles = registrarAccessor.getAllClientIdWithRoles().get(clientId); - changeRegistrarFields(registrar, roles, builder, args); + Registrar updatedRegistrar = registrar; + // Do OWNER only updates to the registrar from the request. + updatedRegistrar = checkAndUpdateOwnerControlledFields(updatedRegistrar, args); + // Do ADMIN only updates to the registrar from the request. + updatedRegistrar = checkAndUpdateAdminControlledFields(updatedRegistrar, args); // read the contacts from the request. ImmutableSet updatedContacts = readContacts(registrar, args); - if (!updatedContacts.isEmpty()) { - builder.setContactsRequireSyncing(true); + + // Save the updated contacts + if (!updatedContacts.equals(contacts)) { + if (!registrarAccessor.hasRoleOnRegistrar(Role.OWNER, registrar.getClientId())) { + throw new ForbiddenException("Only OWNERs can update the contacts"); + } + checkContactRequirements(contacts, updatedContacts); + RegistrarContact.updateContacts(updatedRegistrar, updatedContacts); + updatedRegistrar = + updatedRegistrar.asBuilder().setContactsRequireSyncing(true).build(); } // Save the updated registrar - Registrar updatedRegistrar = builder.build(); if (!updatedRegistrar.equals(registrar)) { ofy().save().entity(updatedRegistrar); } - // Save the updated contacts - if (!updatedContacts.isEmpty()) { - checkContactRequirements(contacts, updatedContacts); - RegistrarContact.updateContacts(updatedRegistrar, updatedContacts); - } - // Email and return update. sendExternalUpdatesIfNecessary( registrar, contacts, updatedRegistrar, updatedContacts); @@ -236,12 +250,15 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA return result; } - /** Updates a registrar builder with the supplied args from the http request; */ - public static void changeRegistrarFields( - Registrar existingRegistrarObj, - Set roles, - Registrar.Builder builder, - Map args) { + /** + * Updates registrar with the OWNER-controlled args from the http request. + * + *

If any changes were made and the user isn't an OWNER - throws a {@link ForbiddenException}. + */ + private Registrar checkAndUpdateOwnerControlledFields( + Registrar initialRegistrar, Map args) { + + Registrar.Builder builder = initialRegistrar.asBuilder(); // BILLING RegistrarFormFields.PREMIUM_PRICE_ACK_REQUIRED @@ -249,13 +266,27 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA .ifPresent(builder::setPremiumPriceAckRequired); // WHOIS - builder.setWhoisServer( - RegistrarFormFields.WHOIS_SERVER_FIELD.extractUntyped(args).orElse(null)); + // + // Because of how whoisServer handles "default value", it's possible that setting the existing + // value will still change the Registrar. So we first check whether the value has changed. + // + // The problem is - if the Registrar has a "null" whoisServer value, the console gets the + // "default value" instead of the actual (null) value. + // This was done so we display the "default" value, but it also means that it always looks like + // the user updated the whoisServer value from "null" to the default value. + // + // TODO(b/119913848):once a null whoisServer value is sent to the console as "null", there's no + // need to check for equality before setting the value in the builder. + String updatedWhoisServer = + RegistrarFormFields.WHOIS_SERVER_FIELD.extractUntyped(args).orElse(null); + if (!Objects.equals(initialRegistrar.getWhoisServer(), updatedWhoisServer)) { + builder.setWhoisServer(updatedWhoisServer); + } builder.setUrl(RegistrarFormFields.URL_FIELD.extractUntyped(args).orElse(null)); // If the email is already null / empty - we can keep it so. But if it's set - it's required to // remain set. - (Strings.isNullOrEmpty(existingRegistrarObj.getEmailAddress()) + (Strings.isNullOrEmpty(initialRegistrar.getEmailAddress()) ? RegistrarFormFields.EMAIL_ADDRESS_FIELD_OPTIONAL : RegistrarFormFields.EMAIL_ADDRESS_FIELD_REQUIRED) .extractUntyped(args) @@ -282,17 +313,59 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA certificate -> builder.setFailoverClientCertificate(certificate, ofy().getTransactionTime())); - // Update allowed TLDs only when it is modified + return checkNotChangedUnlessAllowed(builder, initialRegistrar, Role.OWNER); + } + + /** + * Updates a registrar with the ADMIN-controlled args from the http request. + * + *

If any changes were made and the user isn't an ADMIN - throws a {@link ForbiddenException}. + */ + private Registrar checkAndUpdateAdminControlledFields( + Registrar initialRegistrar, Map args) { + Registrar.Builder builder = initialRegistrar.asBuilder(); + Set updatedAllowedTlds = RegistrarFormFields.ALLOWED_TLDS_FIELD.extractUntyped(args).orElse(ImmutableSet.of()); - if (!updatedAllowedTlds.equals(existingRegistrarObj.getAllowedTlds())) { - // Only admin is allowed to update allowed TLDs - if (roles.contains(Role.ADMIN)) { - builder.setAllowedTlds(updatedAllowedTlds); - } else { - throw new ForbiddenException("Only admin can update allowed TLDs."); - } + // Temporarily block anyone from removing an allowed TLD. + // This is so we can start having Support users use the console in production before we finish + // implementing configurable access control. + // TODO(b/119549884): remove this code once configurable access control is implemented. + if (!Sets.difference(initialRegistrar.getAllowedTlds(), updatedAllowedTlds).isEmpty()) { + throw new ForbiddenException("Can't remove allowed TLDs using the console."); } + builder.setAllowedTlds(updatedAllowedTlds); + return checkNotChangedUnlessAllowed(builder, initialRegistrar, Role.ADMIN); + } + + /** + * Makes sure builder.build is different than originalRegistrar only if we have the correct role. + * + *

On success, returns {@code builder.build()}. + */ + private Registrar checkNotChangedUnlessAllowed( + Registrar.Builder builder, + Registrar originalRegistrar, + Role allowedRole) { + Registrar updatedRegistrar = builder.build(); + if (updatedRegistrar.equals(originalRegistrar)) { + return updatedRegistrar; + } + checkArgument( + updatedRegistrar.getClientId().equals(originalRegistrar.getClientId()), + "Can't change clientId (%s -> %s)", + originalRegistrar.getClientId(), + updatedRegistrar.getClientId()); + if (registrarAccessor.hasRoleOnRegistrar(allowedRole, originalRegistrar.getClientId())) { + return updatedRegistrar; + } + Map diffs = + DiffUtils.deepDiff( + originalRegistrar.toDiffableFieldMap(), + updatedRegistrar.toDiffableFieldMap(), + true); + throw new ForbiddenException( + String.format("Unauthorized: only %s can change fields %s", allowedRole, diffs.keySet())); } /** Reads the contacts from the supplied args. */ @@ -403,7 +476,7 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA ImmutableSet existingContacts, Registrar updatedRegistrar, ImmutableSet updatedContacts) { - if (registrarChangesNotificationEmailAddresses.isEmpty()) { + if (!sendEmailUtils.hasRecepients()) { return; } @@ -420,7 +493,6 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA enqueueRegistrarSheetSync(appEngineServiceUtils.getCurrentVersionHostname("backend")); String environment = Ascii.toLowerCase(String.valueOf(registryEnvironment)); sendEmailUtils.sendEmail( - registrarChangesNotificationEmailAddresses, String.format( "Registrar %s (%s) updated in %s", existingRegistrar.getRegistrarName(), diff --git a/java/google/registry/ui/soy/Console.soy b/java/google/registry/ui/soy/Console.soy index 1619f27ed..7a1bec533 100644 --- a/java/google/registry/ui/soy/Console.soy +++ b/java/google/registry/ui/soy/Console.soy @@ -102,27 +102,25 @@ /** Appbar add/back, edit/cancel appbar. */ {template .appbarButtons} -

- +
+ -
- - -
-
- - -
+ class="{css('kd-button')} {css('kd-button-submit')}"> + Edit + +
+
+ +
{/template} diff --git a/java/google/registry/ui/soy/Forms.soy b/java/google/registry/ui/soy/Forms.soy index ea3a0accd..ae1199545 100644 --- a/java/google/registry/ui/soy/Forms.soy +++ b/java/google/registry/ui/soy/Forms.soy @@ -295,6 +295,21 @@ {/template} +/** Submit button. */ +{template .submitRow} + {@param label: string} + + + + + + +{/template} + /** Drop-down select button widget. */ {template .menuButton} diff --git a/java/google/registry/ui/soy/otesetup/BUILD b/java/google/registry/ui/soy/otesetup/BUILD new file mode 100644 index 000000000..f19b6651c --- /dev/null +++ b/java/google/registry/ui/soy/otesetup/BUILD @@ -0,0 +1,21 @@ +package( + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +load("@io_bazel_rules_closure//closure:defs.bzl", "closure_java_template_library", "closure_js_template_library") + +closure_js_template_library( + name = "otesetup", + srcs = glob(["*.soy"]), + data = ["//java/google/registry/ui/css:registrar_raw"], + globals = "//java/google/registry/ui:globals.txt", + deps = ["//java/google/registry/ui/soy"], +) + +closure_java_template_library( + name = "soy_java_wrappers", + srcs = glob(["*.soy"]), + java_package = "google.registry.ui.soy.otesetup", +) diff --git a/java/google/registry/ui/soy/otesetup/Console.soy b/java/google/registry/ui/soy/otesetup/Console.soy new file mode 100644 index 000000000..10a477e6d --- /dev/null +++ b/java/google/registry/ui/soy/otesetup/Console.soy @@ -0,0 +1,190 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{namespace registry.soy.registrar.console} + + +/** + * Main page for the OT&E creation. Holds a form with the required data. + */ +{template .formPage} + {@param xsrfToken: string} /** Security token. */ + {@param username: string} /** Arbitrary username to display. */ + {@param logoutUrl: string} /** Generated URL for logging out of Google. */ + {@param productName: string} /** Name to display for this software product. */ + {@param logoFilename: string} + {@param? errorMessage: string} /** If set - display the error message above the form. */ + {@param? baseClientId: string} /** If set - an initial value to fill for the base client ID. */ + {@param? contactEmail: string} /** If set - an initial value to fill for the email. */ + + {call registry.soy.console.header} + {param app: 'registrar' /} + {param subtitle: 'OT&E setup' /} + {/call} + {call registry.soy.console.googlebar data="all" /} +
+
+

OT&E Setup Page

+ {if isNonnull($errorMessage)} +

Failed: {$errorMessage}

+ {/if} + {call .form_ data="all" /} +
+ {call registry.soy.console.footer /} +
+{/template} + + +/** + * Result page for a successful OT&E setup. + */ +{template .resultSuccess} + {@param baseClientId: string} /** The base clientId used for the OT&E setup. */ + {@param contactEmail: string} /** The contact's email added to the registrars. */ + {@param clientIdToTld: map} /** The created registrars->TLD mapping. */ + {@param password: string} /** The password given for the created registrars. */ + {@param username: string} /** Arbitrary username to display. */ + {@param logoutUrl: string} /** Generated URL for logging out of Google. */ + {@param productName: string} /** Name to display for this software product. */ + {@param logoFilename: string} + + {call registry.soy.console.header} + {param app: 'registrar' /} + {param subtitle: 'OT&E setup' /} + {/call} + {call registry.soy.console.googlebar data="all" /} +
+
+

OT&E successfully created for registrar {$baseClientId}!

+ + + + +
+ + Copy and paste this into an email to the registrars + + +
+ Gave web-console access to these registrars. +

Don't forget to set the and for these Registrars!

+ Links to the security page for your convenience:
+ {for $clientId in mapKeys($clientIdToTld)} + {$clientId}
+ {/for} +
+ {call registry.soy.console.footer /} +
+{/template} + + +/** Form for getting registrar info. */ +{template .form_ visibility="private"} + {@param xsrfToken: string} /** Security token. */ + {@param? baseClientId: string} /** If set - an initial value to fill for the base client ID. */ + {@param? contactEmail: string} /** If set - an initial value to fill for the email. */ +
+ + + + + + {call registry.soy.forms.inputFieldRowWithValue} + {param label: 'Contact email' /} + {param name: 'email' /} + {param value: $contactEmail /} + {param placeholder: 'registrar\'s assigned email' /} + {param description kind="text"} + Will be granted web-console access to the OTE registrars. + {/param} + {param readonly: false /} + {/call} + {call registry.soy.forms.inputFieldRowWithValue} + {param label: 'Password (optional)' /} + {param name: 'password' /} + {param placeholder: 'Optional password' /} + {param description kind="text"} + The password used to for EPP login. Leave blank to auto-generate a password. + {/param} + {param readonly: false /} + {/call} + {call registry.soy.forms.submitRow} + {param label: 'create' /} + {/call} +
+ {call registry.soy.forms.inputFieldLabel} + {param label: 'Base client ID' /} + {/call} + + {call registry.soy.forms.inputFieldValue } + {param name: 'clientId' /} + {param value: $baseClientId /} + {param placeholder: 'registrar\'s ID' /} + {param readonly: false /} + {/call} + + Must: +
    +
  1. consist of only lowercase letters, numbers, or hyphens,
    2. +
  2. start with a letter, and
    3. +
  3. be between 3 and 14 characters (inclusive).
    4. +
+ We require 1 and 2 since the registrar name will be used to create TLDs, and we + require 3 since we append "-[12345]" to the name to create client IDs which are + required by the EPP XML schema to be between 3-16 chars. + +
+ +
+{/template} + + +/** + * Who goes thar?! + */ +{template .whoareyou} + {@param username: string} /** Arbitrary username to display. */ + {@param logoutUrl: string} /** Generated URL for logging out of Google. */ + {@param logoFilename: string} + {@param productName: string} + {call registry.soy.console.header} + {param app: 'registrar' /} + {param subtitle: 'Not Authorized' /} + {/call} +
+ + {$productName} + +

You need permission

+

+ Only {$productName} Admins have access to this page. + You are signed in as {$username}. +

+
+{/template} + + diff --git a/java/google/registry/ui/soy/registrar/AdminSettings.soy b/java/google/registry/ui/soy/registrar/AdminSettings.soy new file mode 100644 index 000000000..d9e18993e --- /dev/null +++ b/java/google/registry/ui/soy/registrar/AdminSettings.soy @@ -0,0 +1,66 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{namespace registry.soy.registrar.admin} + + + +/** Registrar admin settings page for view and edit. */ +{template .settings} + {@param allowedTlds: list} +
+

Administrator settings

+ + + +
+ + set or remove TLDs this + client is allowed access to. + +
+
+ {for $tld in $allowedTlds} + {call .tld} + {param name: 'allowedTlds[' + index($tld) + ']' /} + {param tld: $tld /} + {/call} + {/for} +
+
+ + +
+
+
+ + +

Generate new OT&E accounts here +

+
+{/template} + + +/** TLD form input. */ +{template .tld} + {@param name: string} + {@param tld: string} +
+ + +
+{/template} diff --git a/java/google/registry/ui/soy/registrar/Console.soy b/java/google/registry/ui/soy/registrar/Console.soy index 65ef43d21..a66bdeafd 100644 --- a/java/google/registry/ui/soy/registrar/Console.soy +++ b/java/google/registry/ui/soy/registrar/Console.soy @@ -24,6 +24,8 @@ {@param xsrfToken: string} /** Security token. */ {@param clientId: string} /** Registrar client identifier. */ {@param allClientIds: list} /** All registrar client identifiers for the user. */ + {@param isAdmin: bool} + {@param isOwner: bool} {@param username: string} /** Arbitrary username to display. */ {@param logoutUrl: string} /** Generated URL for logging out of Google. */ {@param productName: string} /** Name to display for this software product. */ @@ -39,7 +41,17 @@ {/call} {call registry.soy.console.googlebar data="all" /}
-
+
+
+ Accessing {$clientId} as{sp} + {if $isOwner}Owner{/if} + {if $isAdmin}Admin{/if} + {if length($allClientIds) > 1} + {sp}(Switch registrar: {call .clientIdSelect_ data="all" /}) + {/if} +
+
+
{call .navbar_ data="all" /}
@@ -55,7 +67,7 @@ {case google.registry.ui.ConsoleDebug.RAW} - + {/switch} @@ -63,6 +75,8 @@