diff --git a/buildSrc/gradle/dependency-locks/compile.lockfile b/buildSrc/gradle/dependency-locks/compile.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/compile.lockfile +++ b/buildSrc/gradle/dependency-locks/compile.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/compileClasspath.lockfile b/buildSrc/gradle/dependency-locks/compileClasspath.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/compileClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/compileClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/deploy_jar.lockfile b/buildSrc/gradle/dependency-locks/deploy_jar.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/deploy_jar.lockfile +++ b/buildSrc/gradle/dependency-locks/deploy_jar.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/runtime.lockfile b/buildSrc/gradle/dependency-locks/runtime.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/runtime.lockfile +++ b/buildSrc/gradle/dependency-locks/runtime.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile b/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/testCompile.lockfile b/buildSrc/gradle/dependency-locks/testCompile.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testCompile.lockfile +++ b/buildSrc/gradle/dependency-locks/testCompile.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile b/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/buildSrc/gradle/dependency-locks/testRuntime.lockfile b/buildSrc/gradle/dependency-locks/testRuntime.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testRuntime.lockfile +++ b/buildSrc/gradle/dependency-locks/testRuntime.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile b/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/common/gradle/dependency-locks/compile.lockfile b/common/gradle/dependency-locks/compile.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/compile.lockfile +++ b/common/gradle/dependency-locks/compile.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/compileClasspath.lockfile b/common/gradle/dependency-locks/compileClasspath.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/compileClasspath.lockfile +++ b/common/gradle/dependency-locks/compileClasspath.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/default.lockfile b/common/gradle/dependency-locks/default.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/default.lockfile +++ b/common/gradle/dependency-locks/default.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/deploy_jar.lockfile b/common/gradle/dependency-locks/deploy_jar.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/deploy_jar.lockfile +++ b/common/gradle/dependency-locks/deploy_jar.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/runtime.lockfile b/common/gradle/dependency-locks/runtime.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/runtime.lockfile +++ b/common/gradle/dependency-locks/runtime.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/runtimeClasspath.lockfile b/common/gradle/dependency-locks/runtimeClasspath.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/common/gradle/dependency-locks/runtimeClasspath.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/testCompile.lockfile b/common/gradle/dependency-locks/testCompile.lockfile index 017019006..fbea9547f 100644 --- a/common/gradle/dependency-locks/testCompile.lockfile +++ b/common/gradle/dependency-locks/testCompile.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testCompileClasspath.lockfile b/common/gradle/dependency-locks/testCompileClasspath.lockfile index 017019006..fbea9547f 100644 --- a/common/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/common/gradle/dependency-locks/testCompileClasspath.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testRuntime.lockfile b/common/gradle/dependency-locks/testRuntime.lockfile index 6930dcb16..d45bd7305 100644 --- a/common/gradle/dependency-locks/testRuntime.lockfile +++ b/common/gradle/dependency-locks/testRuntime.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testRuntimeClasspath.lockfile b/common/gradle/dependency-locks/testRuntimeClasspath.lockfile index 6930dcb16..d45bd7305 100644 --- a/common/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/common/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingCompile.lockfile b/common/gradle/dependency-locks/testingCompile.lockfile index c67ff6845..1893cd4b9 100644 --- a/common/gradle/dependency-locks/testingCompile.lockfile +++ b/common/gradle/dependency-locks/testingCompile.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingCompileClasspath.lockfile b/common/gradle/dependency-locks/testingCompileClasspath.lockfile index c67ff6845..1893cd4b9 100644 --- a/common/gradle/dependency-locks/testingCompileClasspath.lockfile +++ b/common/gradle/dependency-locks/testingCompileClasspath.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingRuntime.lockfile b/common/gradle/dependency-locks/testingRuntime.lockfile index 7f20d13e3..fdb906898 100644 --- a/common/gradle/dependency-locks/testingRuntime.lockfile +++ b/common/gradle/dependency-locks/testingRuntime.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile b/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile index 7f20d13e3..fdb906898 100644 --- a/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile +++ b/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/config/presubmits.py b/config/presubmits.py index 442676ea1..9d85c4c3c 100644 --- a/config/presubmits.py +++ b/config/presubmits.py @@ -79,8 +79,8 @@ PRESUBMITS = { r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.", ("java", "js", "soy", "sql", "py", "sh", "gradle"), { ".git", "/build/", "/generated/", "/generated_tests/", - "node_modules/", "JUnitBackports.java", "registrar_bin.", - "registrar_dbg.", "google-java-format-diff.py", + "node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java", + "registrar_bin.", "registrar_dbg.", "google-java-format-diff.py", "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js" }, REQUIRED): "File did not include the license header.", diff --git a/core/build.gradle b/core/build.gradle index ca7809029..f280b0f99 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -197,6 +197,7 @@ dependencies { compile deps['com.google.apis:google-api-services-groupssettings'] compile deps['com.google.apis:google-api-services-monitoring'] compile deps['com.google.apis:google-api-services-sheets'] + compile deps['com.google.apis:google-api-services-storage'] testCompile deps['com.google.appengine:appengine-api-stubs'] compile deps['com.google.appengine.tools:appengine-gcs-client'] compile deps['com.google.appengine.tools:appengine-mapreduce'] @@ -221,6 +222,8 @@ dependencies { gradleLint.ignore('unused-dependency') { compile deps['com.google.gwt:gwt-user'] } + compile deps['com.google.cloud:google-cloud-core'] + compile deps['com.google.cloud:google-cloud-storage'] compile deps['com.google.http-client:google-http-client'] compile deps['com.google.http-client:google-http-client-appengine'] compile deps['com.google.http-client:google-http-client-jackson2'] @@ -315,6 +318,7 @@ dependencies { annotationProcessor project(':processor') testAnnotationProcessor project(':processor') + testCompile deps['com.google.cloud:google-cloud-nio'] testCompile deps['com.google.appengine:appengine-testing'] testCompile deps['com.google.guava:guava-testlib'] testCompile deps['com.google.monitoring-client:contrib'] diff --git a/core/gradle/dependency-locks/annotationProcessor.lockfile b/core/gradle/dependency-locks/annotationProcessor.lockfile index f0a2ecd5d..09dbb3924 100644 --- a/core/gradle/dependency-locks/annotationProcessor.lockfile +++ b/core/gradle/dependency-locks/annotationProcessor.lockfile @@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33 com.google.dagger:dagger-spi:2.33 com.google.dagger:dagger:2.33 com.google.errorprone:error_prone_annotation:2.3.4 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.errorprone:error_prone_check_api:2.3.4 com.google.errorprone:error_prone_core:2.3.4 com.google.errorprone:error_prone_type_annotations:2.3.4 com.google.errorprone:javac-shaded:9-dev-r4023-3 com.google.googlejavaformat:google-java-format:1.5 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.protobuf:protobuf-java:3.4.0 @@ -32,7 +32,7 @@ javax.inject:javax.inject:1 javax.persistence:javax.persistence-api:2.2 net.ltgt.gradle.incap:incap:0.2 org.checkerframework:checker-compat-qual:2.5.3 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.checkerframework:dataflow:3.0.0 org.checkerframework:javacutil:3.0.0 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20 diff --git a/core/gradle/dependency-locks/compile.lockfile b/core/gradle/dependency-locks/compile.lockfile index e3e31a555..da39c21db 100644 --- a/core/gradle/dependency-locks/compile.lockfile +++ b/core/gradle/dependency-locks/compile.lockfile @@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/compileClasspath.lockfile b/core/gradle/dependency-locks/compileClasspath.lockfile index 80864edca..1631f4d24 100644 --- a/core/gradle/dependency-locks/compileClasspath.lockfile +++ b/core/gradle/dependency-locks/compileClasspath.lockfile @@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.14.0 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -215,7 +216,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.conscrypt:conscrypt-openjdk-uber:2.5.1 diff --git a/core/gradle/dependency-locks/default.lockfile b/core/gradle/dependency-locks/default.lockfile index 350bc7d70..d8072a9ef 100644 --- a/core/gradle/dependency-locks/default.lockfile +++ b/core/gradle/dependency-locks/default.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/deploy_jar.lockfile b/core/gradle/dependency-locks/deploy_jar.lockfile index d67debded..c3b897de3 100644 --- a/core/gradle/dependency-locks/deploy_jar.lockfile +++ b/core/gradle/dependency-locks/deploy_jar.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/nonprodCompile.lockfile b/core/gradle/dependency-locks/nonprodCompile.lockfile index e3e31a555..da39c21db 100644 --- a/core/gradle/dependency-locks/nonprodCompile.lockfile +++ b/core/gradle/dependency-locks/nonprodCompile.lockfile @@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile b/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile index 70618ba77..c28288d48 100644 --- a/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile +++ b/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile @@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.14.0 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -216,7 +217,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.conscrypt:conscrypt-openjdk-uber:2.5.1 diff --git a/core/gradle/dependency-locks/nonprodRuntime.lockfile b/core/gradle/dependency-locks/nonprodRuntime.lockfile index 36adf8801..42f0cec4d 100644 --- a/core/gradle/dependency-locks/nonprodRuntime.lockfile +++ b/core/gradle/dependency-locks/nonprodRuntime.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile b/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile index 36adf8801..42f0cec4d 100644 --- a/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile +++ b/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/runtime.lockfile b/core/gradle/dependency-locks/runtime.lockfile index 36adf8801..42f0cec4d 100644 --- a/core/gradle/dependency-locks/runtime.lockfile +++ b/core/gradle/dependency-locks/runtime.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/runtimeClasspath.lockfile b/core/gradle/dependency-locks/runtimeClasspath.lockfile index d67debded..c3b897de3 100644 --- a/core/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/core/gradle/dependency-locks/runtimeClasspath.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/testAnnotationProcessor.lockfile b/core/gradle/dependency-locks/testAnnotationProcessor.lockfile index 3cb0bf2ea..69c9128fc 100644 --- a/core/gradle/dependency-locks/testAnnotationProcessor.lockfile +++ b/core/gradle/dependency-locks/testAnnotationProcessor.lockfile @@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33 com.google.dagger:dagger-spi:2.33 com.google.dagger:dagger:2.33 com.google.errorprone:error_prone_annotation:2.3.4 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.errorprone:error_prone_check_api:2.3.4 com.google.errorprone:error_prone_core:2.3.4 com.google.errorprone:error_prone_type_annotations:2.3.4 com.google.errorprone:javac-shaded:9-dev-r4023-3 com.google.googlejavaformat:google-java-format:1.5 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.protobuf:protobuf-java:3.4.0 @@ -30,7 +30,7 @@ javax.inject:javax.inject:1 javax.persistence:javax.persistence-api:2.2 net.ltgt.gradle.incap:incap:0.2 org.checkerframework:checker-compat-qual:2.5.3 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.checkerframework:dataflow:3.0.0 org.checkerframework:javacutil:3.0.0 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20 diff --git a/core/gradle/dependency-locks/testCompile.lockfile b/core/gradle/dependency-locks/testCompile.lockfile index 8e64dd0b4..fbbece5be 100644 --- a/core/gradle/dependency-locks/testCompile.lockfile +++ b/core/gradle/dependency-locks/testCompile.lockfile @@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0 args4j:args4j:2.0.23 cglib:cglib-nodep:2.2 com.beust:jcommander:1.60 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -54,12 +54,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -84,10 +84,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -98,32 +98,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -135,9 +137,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -161,7 +163,7 @@ io.github.classgraph:classgraph:4.8.102 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -199,7 +201,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 net.java.dev.jna:jna:5.5.0 @@ -307,7 +309,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.xerial.snappy:snappy-java:1.1.4 diff --git a/core/gradle/dependency-locks/testCompileClasspath.lockfile b/core/gradle/dependency-locks/testCompileClasspath.lockfile index d112fcc04..270390b42 100644 --- a/core/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/core/gradle/dependency-locks/testCompileClasspath.lockfile @@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0 args4j:args4j:2.0.23 cglib:cglib-nodep:2.2 com.beust:jcommander:1.60 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -53,12 +53,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -83,10 +83,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -97,31 +97,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -133,9 +135,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.14.0 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -159,7 +161,7 @@ io.github.classgraph:classgraph:4.8.102 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -194,7 +196,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 net.java.dev.jna:jna:5.5.0 @@ -301,7 +303,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.xerial.snappy:snappy-java:1.1.4 diff --git a/core/gradle/dependency-locks/testRuntime.lockfile b/core/gradle/dependency-locks/testRuntime.lockfile index ca0377997..5096bf38c 100644 --- a/core/gradle/dependency-locks/testRuntime.lockfile +++ b/core/gradle/dependency-locks/testRuntime.lockfile @@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.arnx:nashorn-promise:0.1.1 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 @@ -320,7 +322,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.webjars.npm:viz.js-for-graphviz-java:2.1.3 diff --git a/core/gradle/dependency-locks/testRuntimeClasspath.lockfile b/core/gradle/dependency-locks/testRuntimeClasspath.lockfile index e78acec00..d9d835407 100644 --- a/core/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/core/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.arnx:nashorn-promise:0.1.1 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 @@ -321,7 +323,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.webjars.npm:viz.js-for-graphviz-java:2.1.3 diff --git a/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java b/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java index c23d59372..767326880 100644 --- a/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java +++ b/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java @@ -28,18 +28,17 @@ import static google.registry.model.ofy.CommitLogBucket.getBucketKey; import static google.registry.model.ofy.ObjectifyService.auditedOfy; import static google.registry.util.DateTimeUtils.START_OF_TIME; import static google.registry.util.DateTimeUtils.isAtOrAfter; -import static java.nio.channels.Channels.newOutputStream; import static java.util.Comparator.comparingLong; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; import com.googlecode.objectify.Key; import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; import google.registry.model.ImmutableObject; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; @@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable { public static final String DIFF_FILE_PREFIX = "commit_diff_until_"; - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; + @Inject @Config("commitLogGcsBucket") String gcsBucket; @Inject @Config("commitLogDiffExportBatchSize") int batchSize; @Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime; @@ -102,13 +102,13 @@ public final class ExportCommitLogDiffAction implements Runnable { List> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint); logger.atInfo().log("Found %d manifests to export", sortedKeys.size()); // Open an output channel to GCS, wrapped in a stream for convenience. - try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace( - new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString()) - .addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString()) - .addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())) - .build()))) { + try (OutputStream gcsStream = + gcsUtils.openOutputStream( + BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime), + ImmutableMap.of( + LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(), + UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(), + NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) { // Export the upper checkpoint itself. serializeEntity(upperCheckpoint, gcsStream); // If there are no manifests to export, stop early, now that we've written out the file with diff --git a/core/src/main/java/google/registry/backup/GcsDiffFileLister.java b/core/src/main/java/google/registry/backup/GcsDiffFileLister.java index 7fd887a8e..19305a9e2 100644 --- a/core/src/main/java/google/registry/backup/GcsDiffFileLister.java +++ b/core/src/main/java/google/registry/backup/GcsDiffFileLister.java @@ -21,19 +21,16 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME; import static google.registry.util.DateTimeUtils.isBeforeOrAt; import static google.registry.util.DateTimeUtils.latestOf; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.ListItem; -import com.google.appengine.tools.cloudstorage.ListOptions; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import google.registry.backup.BackupModule.Backups; +import google.registry.gcs.GcsUtils; import java.io.IOException; -import java.util.Iterator; import java.util.Map; import java.util.TreeMap; import javax.annotation.Nullable; @@ -45,7 +42,8 @@ class GcsDiffFileLister { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; + @Inject @Backups ListeningExecutorService executor; @Inject GcsDiffFileLister() {} @@ -56,22 +54,22 @@ class GcsDiffFileLister { */ private boolean constructDiffSequence( String gcsBucket, - Map> upperBoundTimesToMetadata, + Map> upperBoundTimesToBlobInfo, DateTime fromTime, DateTime lastTime, - TreeMap sequence) { + TreeMap sequence) { DateTime checkpointTime = lastTime; while (isBeforeOrAt(fromTime, checkpointTime)) { - GcsFileMetadata metadata; - if (upperBoundTimesToMetadata.containsKey(checkpointTime)) { - metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime)); + BlobInfo blobInfo; + if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) { + blobInfo = Futures.getUnchecked(upperBoundTimesToBlobInfo.get(checkpointTime)); } else { String filename = DIFF_FILE_PREFIX + checkpointTime; logger.atInfo().log("Patching GCS list; discovered file: %s", filename); - metadata = getMetadata(gcsBucket, filename); + blobInfo = getBlobInfo(gcsBucket, filename); // If we hit a gap, quit. - if (metadata == null) { + if (blobInfo == null) { logger.atInfo().log( "Gap discovered in sequence terminating at %s, missing file: %s", sequence.lastKey(), filename); @@ -79,14 +77,14 @@ class GcsDiffFileLister { return false; } } - sequence.put(checkpointTime, metadata); - checkpointTime = getLowerBoundTime(metadata); + sequence.put(checkpointTime, blobInfo); + checkpointTime = getLowerBoundTime(blobInfo); } logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime); return true; } - ImmutableList listDiffFiles( + ImmutableList listDiffFiles( String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) { logger.atInfo().log("Requested restore from time: %s", fromTime); if (toTime != null) { @@ -95,28 +93,25 @@ class GcsDiffFileLister { // List all of the diff files on GCS and build a map from each file's upper checkpoint time // (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with // an upper checkpoint time > fromTime. - TreeMap> upperBoundTimesToMetadata - = new TreeMap<>(); - Iterator listItems; + TreeMap> upperBoundTimesToBlobInfo = new TreeMap<>(); + ImmutableList strippedFilenames; try { - // TODO(b/23554360): Use a smarter prefixing strategy to speed this up. - listItems = gcsService.list( - gcsBucket, - new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build()); + strippedFilenames = gcsUtils.listFolderObjects(gcsBucket, DIFF_FILE_PREFIX); } catch (IOException e) { throw new RuntimeException(e); } DateTime lastUpperBoundTime = START_OF_TIME; - while (listItems.hasNext()) { - final String filename = listItems.next().getName(); - DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length())); + + for (String strippedFilename : strippedFilenames) { + final String filename = DIFF_FILE_PREFIX + strippedFilename; + DateTime upperBoundTime = DateTime.parse(strippedFilename); if (isInRange(upperBoundTime, fromTime, toTime)) { - upperBoundTimesToMetadata.put( - upperBoundTime, executor.submit(() -> getMetadata(gcsBucket, filename))); + upperBoundTimesToBlobInfo.put( + upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename))); lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime); } } - if (upperBoundTimesToMetadata.isEmpty()) { + if (upperBoundTimesToBlobInfo.isEmpty()) { logger.atInfo().log("No files found"); return ImmutableList.of(); } @@ -129,23 +124,23 @@ class GcsDiffFileLister { // metadata of a file is sufficient to identify the preceding file, so if we start from the // last file and work backwards we can verify that we have no holes in our chain (although we // may be missing files at the end). - TreeMap sequence = new TreeMap<>(); + TreeMap sequence = new TreeMap<>(); logger.atInfo().log("Restoring until: %s", lastUpperBoundTime); boolean inconsistentFileSet = !constructDiffSequence( - gcsBucket, upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence); + gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence); // Verify that all of the elements in the original set are represented in the sequence. If we // find anything that's not represented, construct a sequence for it. boolean checkForMoreExtraDiffs = true; // Always loop at least once. while (checkForMoreExtraDiffs) { checkForMoreExtraDiffs = false; - for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) { + for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) { if (!isInRange(key, fromTime, toTime)) { break; } if (!sequence.containsKey(key)) { - constructDiffSequence(gcsBucket, upperBoundTimesToMetadata, fromTime, key, sequence); + constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence); checkForMoreExtraDiffs = true; inconsistentFileSet = true; break; @@ -173,15 +168,11 @@ class GcsDiffFileLister { return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end)); } - private DateTime getLowerBoundTime(GcsFileMetadata metadata) { - return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT)); + private DateTime getLowerBoundTime(BlobInfo blobInfo) { + return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT)); } - private GcsFileMetadata getMetadata(String gcsBucket, String filename) { - try { - return gcsService.getMetadata(new GcsFilename(gcsBucket, filename)); - } catch (IOException e) { - throw new RuntimeException(e); - } + private BlobInfo getBlobInfo(String gcsBucket, String filename) { + return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename)); } } diff --git a/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java b/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java index 5709ab07e..8bcb1c0c3 100644 --- a/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java +++ b/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java @@ -25,12 +25,12 @@ import static org.joda.time.Duration.standardHours; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.Key; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobInfo; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; import google.registry.model.common.DatabaseMigrationStateSchedule; import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection; @@ -51,7 +51,6 @@ import google.registry.util.Clock; import google.registry.util.RequestStatusChecker; import java.io.IOException; import java.io.InputStream; -import java.nio.channels.Channels; import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletResponse; @@ -70,14 +69,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable { static final String PATH = "/_dr/task/replayCommitLogsToSql"; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final int BLOCK_SIZE = - 1024 * 1024; // Buffer 1mb at a time, for no particular reason. + private static final Duration LEASE_LENGTH = standardHours(1); // Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE // request timeouts private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5); - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; @Inject Response response; @Inject RequestStatusChecker requestStatusChecker; @Inject GcsDiffFileLister diffLister; @@ -122,13 +120,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable { } try { logger.atInfo().log("Beginning replay of commit logs."); - ImmutableList commitLogFiles = getFilesToReplay(); + ImmutableList commitLogFiles = getFilesToReplay(); if (dryRun) { response.setStatus(HttpServletResponse.SC_OK); ImmutableList filenames = commitLogFiles.stream() .limit(10) - .map(file -> file.getFilename().getObjectName()) + .map(file -> file.getName()) .collect(toImmutableList()); String dryRunMessage = "Running in dry-run mode; would have processed %d files. They are (limit 10):\n" @@ -152,22 +150,22 @@ public class ReplayCommitLogsToSqlAction implements Runnable { } } - private ImmutableList getFilesToReplay() { + private ImmutableList getFilesToReplay() { // Start at the first millisecond we haven't seen yet DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1)); logger.atInfo().log("Starting replay from: %s.", fromTime); // If there's an inconsistent file set, this will throw IllegalStateException and the job // will try later -- this is likely because an export hasn't finished yet. - ImmutableList commitLogFiles = + ImmutableList commitLogFiles = diffLister.listDiffFiles(gcsBucket, fromTime, /* current time */ null); logger.atInfo().log("Found %d new commit log files to process.", commitLogFiles.size()); return commitLogFiles; } - private void replayFiles(ImmutableList commitLogFiles) { + private void replayFiles(ImmutableList commitLogFiles) { DateTime replayTimeoutTime = clock.nowUtc().plus(REPLAY_TIMEOUT_DURATION); int processedFiles = 0; - for (GcsFileMetadata metadata : commitLogFiles) { + for (BlobInfo metadata : commitLogFiles) { // One transaction per GCS file jpaTm().transact(() -> processFile(metadata)); processedFiles++; @@ -181,31 +179,26 @@ public class ReplayCommitLogsToSqlAction implements Runnable { logger.atInfo().log("Replayed %d commit log files to SQL successfully.", processedFiles); } - private void processFile(GcsFileMetadata metadata) { + private void processFile(BlobInfo metadata) { logger.atInfo().log( - "Processing commit log file %s of size %d B.", - metadata.getFilename(), metadata.getLength()); - try (InputStream input = - Channels.newInputStream( - gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) { + "Processing commit log file %s of size %d B.", metadata.getName(), metadata.getSize()); + try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) { // Load and process the Datastore transactions one at a time ImmutableList> allTransactions = CommitLogImports.loadEntitiesByTransaction(input); logger.atInfo().log( "Replaying %d transactions from commit log file %s.", - allTransactions.size(), metadata.getFilename()); + allTransactions.size(), metadata.getName()); allTransactions.forEach(this::replayTransaction); // if we succeeded, set the last-seen time - DateTime checkpoint = - DateTime.parse( - metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length())); + DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length())); SqlReplayCheckpoint.set(checkpoint); logger.atInfo().log( "Replayed %d transactions from commit log file %s.", - allTransactions.size(), metadata.getFilename()); + allTransactions.size(), metadata.getName()); } catch (IOException e) { throw new RuntimeException( - "Errored out while replaying commit log file " + metadata.getFilename(), e); + "Errored out while replaying commit log file " + metadata.getName(), e); } } diff --git a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java index fdf99f69f..be8130692 100644 --- a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java +++ b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java @@ -23,8 +23,7 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy; import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.EntityTranslator; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobInfo; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -36,6 +35,7 @@ import com.googlecode.objectify.Result; import com.googlecode.objectify.util.ResultNow; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; +import google.registry.gcs.GcsUtils; import google.registry.model.ImmutableObject; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; @@ -48,7 +48,6 @@ import google.registry.request.auth.Auth; import google.registry.util.Retrier; import java.io.IOException; import java.io.InputStream; -import java.nio.channels.Channels; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,8 +68,6 @@ public class RestoreCommitLogsAction implements Runnable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - static final int BLOCK_SIZE = 1024 * 1024; // Buffer 1mb at a time, for no particular reason. - public static final String PATH = "/_dr/task/restoreCommitLogs"; static final String DRY_RUN_PARAM = "dryRun"; static final String FROM_TIME_PARAM = "fromTime"; @@ -80,7 +77,8 @@ public class RestoreCommitLogsAction implements Runnable { private static final ImmutableSet FORBIDDEN_ENVIRONMENTS = ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX); - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; + @Inject @Parameter(DRY_RUN_PARAM) boolean dryRun; @Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime; @Inject @Parameter(TO_TIME_PARAM) DateTime toTime; @@ -109,17 +107,16 @@ public class RestoreCommitLogsAction implements Runnable { } String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket); logger.atInfo().log("Restoring from %s.", gcsBucket); - List diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime); + List diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime); if (diffFiles.isEmpty()) { logger.atInfo().log("Nothing to restore"); return; } Map bucketTimestamps = new HashMap<>(); CommitLogCheckpoint lastCheckpoint = null; - for (GcsFileMetadata metadata : diffFiles) { - logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName()); - try (InputStream input = Channels.newInputStream( - gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) { + for (BlobInfo metadata : diffFiles) { + logger.atInfo().log("Restoring: %s", metadata.getName()); + try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) { PeekingIterator commitLogs = peekingIterator(createDeserializingIterator(input, true)); lastCheckpoint = (CommitLogCheckpoint) commitLogs.next(); diff --git a/core/src/main/java/google/registry/config/RegistryConfig.java b/core/src/main/java/google/registry/config/RegistryConfig.java index e95ce1bc0..b8de3f0d8 100644 --- a/core/src/main/java/google/registry/config/RegistryConfig.java +++ b/core/src/main/java/google/registry/config/RegistryConfig.java @@ -416,17 +416,6 @@ public final class RegistryConfig { return Optional.ofNullable(config.cloudDns.servicePath); } - /** - * Returns size of Google Cloud Storage client connection buffer in bytes. - * - * @see google.registry.gcs.GcsUtils - */ - @Provides - @Config("gcsBufferSize") - public static int provideGcsBufferSize() { - return 1024 * 1024; - } - /** * Returns the email address of the admin account on the G Suite app used to perform * administrative actions. diff --git a/core/src/main/java/google/registry/export/ExportDomainListsAction.java b/core/src/main/java/google/registry/export/ExportDomainListsAction.java index 6b69fdb83..9774323af 100644 --- a/core/src/main/java/google/registry/export/ExportDomainListsAction.java +++ b/core/src/main/java/google/registry/export/ExportDomainListsAction.java @@ -14,7 +14,6 @@ package google.registry.export; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.base.Verify.verifyNotNull; import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; import static google.registry.model.EppResourceUtils.isActive; @@ -24,11 +23,10 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.request.Action.Method.POST; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.RetryParams; import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Suppliers; @@ -79,9 +77,9 @@ public class ExportDomainListsAction implements Runnable { @Inject Response response; @Inject Clock clock; @Inject DriveConnection driveConnection; + @Inject GcsUtils gcsUtils; @Inject @Config("domainListsGcsBucket") String gcsBucket; - @Inject @Config("gcsBufferSize") int gcsBufferSize; @Inject ExportDomainListsAction() {} @Override @@ -95,7 +93,7 @@ public class ExportDomainListsAction implements Runnable { .setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS)) .runMapreduce( new ExportDomainListsMapper(clock.nowUtc(), realTlds), - new ExportDomainListsReducer(gcsBucket, gcsBufferSize), + new ExportDomainListsReducer(gcsBucket, gcsUtils), ImmutableList.of(createEntityInput(DomainBase.class))) .sendLinkToMapreduceConsole(response); } else { @@ -134,7 +132,7 @@ public class ExportDomainListsAction implements Runnable { String domainsList = Joiner.on("\n").join(domains); logger.atInfo().log( "Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld); - exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize); + exportToGcs(tld, domainsList, gcsBucket, gcsUtils); exportToDrive(tld, domainsList, driveConnection); }); } @@ -168,11 +166,9 @@ public class ExportDomainListsAction implements Runnable { } protected static boolean exportToGcs( - String tld, String domains, String gcsBucket, int gcsBufferSize) { - GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt"); - GcsUtils cloudStorage = - new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize); - try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename); + String tld, String domains, String gcsBucket, GcsUtils gcsUtils) { + BlobId blobId = BlobId.of(gcsBucket, tld + ".txt"); + try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId); Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) { osWriter.write(domains); } catch (Throwable e) { @@ -214,7 +210,7 @@ public class ExportDomainListsAction implements Runnable { Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection()); private final String gcsBucket; - private final int gcsBufferSize; + private final GcsUtils gcsUtils; /** * Non-serializable {@link DriveConnection} that will be created when an instance of {@link @@ -224,9 +220,9 @@ public class ExportDomainListsAction implements Runnable { */ private transient DriveConnection driveConnection; - public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) { + public ExportDomainListsReducer(String gcsBucket, GcsUtils gcsUtils) { this.gcsBucket = gcsBucket; - this.gcsBufferSize = gcsBufferSize; + this.gcsUtils = gcsUtils; } @SuppressWarnings("unused") @@ -240,7 +236,7 @@ public class ExportDomainListsAction implements Runnable { ImmutableList domains = ImmutableList.sortedCopyOf(() -> fqdns); String domainsList = Joiner.on('\n').join(domains); logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld); - if (exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize)) { + if (exportToGcs(tld, domainsList, gcsBucket, gcsUtils)) { getContext().incrementCounter("domain lists successful written out to GCS"); } else { getContext().incrementCounter("domain lists failed to write out to GCS"); diff --git a/core/src/main/java/google/registry/gcs/GcsServiceModule.java b/core/src/main/java/google/registry/gcs/GcsServiceModule.java deleted file mode 100644 index 84dd41b30..000000000 --- a/core/src/main/java/google/registry/gcs/GcsServiceModule.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gcs; - -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; -import com.google.appengine.tools.cloudstorage.RetryParams; -import dagger.Module; -import dagger.Provides; - -/** Dagger module for {@link GcsService}. */ -@Module -public final class GcsServiceModule { - - private static final GcsService gcsService = - GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance()); - - @Provides - static GcsService provideGcsService() { - return gcsService; - } -} diff --git a/core/src/main/java/google/registry/gcs/GcsUtils.java b/core/src/main/java/google/registry/gcs/GcsUtils.java index a159bfb17..7b4ee872c 100644 --- a/core/src/main/java/google/registry/gcs/GcsUtils.java +++ b/core/src/main/java/google/registry/gcs/GcsUtils.java @@ -14,30 +14,39 @@ package google.registry.gcs; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getLast; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.ListOptions; -import com.google.appengine.tools.cloudstorage.ListResult; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; -import google.registry.config.RegistryConfig.Config; +import google.registry.config.CredentialModule.DefaultCredential; +import google.registry.util.GoogleCredentialsBundle; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; +import java.io.Serializable; import java.nio.channels.Channels; import javax.annotation.CheckReturnValue; import javax.inject.Inject; -/** Utilities for working with Google Cloud Storage. */ -public class GcsUtils { +/** + * Utilities for working with Google Cloud Storage. + * + *

It is {@link Serializable} so that it can be used in MapReduce or Beam. + */ +public class GcsUtils implements Serializable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -48,30 +57,65 @@ public class GcsUtils { .put("json", MediaType.JSON_UTF_8) .build(); - private final GcsService gcsService; - private final int bufferSize; + private final StorageOptions storageOptions; - @Inject - public GcsUtils(GcsService gcsService, @Config("gcsBufferSize") int bufferSize) { - this.gcsService = gcsService; - this.bufferSize = bufferSize; + private Storage storage() { + return storageOptions.getService(); } - /** Opens a GCS file for reading as an {@link InputStream} with prefetching. */ + @Inject + public GcsUtils(@DefaultCredential GoogleCredentialsBundle credentialsBundle) { + this( + StorageOptions.newBuilder() + .setCredentials(credentialsBundle.getGoogleCredentials()) + .build()); + } + + @VisibleForTesting + public GcsUtils(StorageOptions storageOptions) { + this.storageOptions = storageOptions; + } + + /** Opens a GCS file for reading as an {@link InputStream}. */ @CheckReturnValue - public InputStream openInputStream(GcsFilename filename) { - return Channels.newInputStream(gcsService.openPrefetchingReadChannel(filename, 0, bufferSize)); + public InputStream openInputStream(BlobId blobId) { + return Channels.newInputStream(storage().reader(blobId)); } /** Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files. */ @CheckReturnValue - public OutputStream openOutputStream(GcsFilename filename) throws IOException { - return Channels.newOutputStream(gcsService.createOrReplace(filename, getOptions(filename))); + public OutputStream openOutputStream(BlobId blobId) { + return Channels.newOutputStream(storage().writer(createBlobInfo(blobId))); + } + + /** + * Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files and setting + * the given metadata. + */ + @CheckReturnValue + public OutputStream openOutputStream(BlobId blobId, ImmutableMap metadata) { + return Channels.newOutputStream( + storage().writer(BlobInfo.newBuilder(blobId).setMetadata(metadata).build())); } /** Creates a GCS file with the given byte contents, overwriting existing files. */ - public void createFromBytes(GcsFilename filename, byte[] bytes) throws IOException { - gcsService.createOrReplace(filename, getOptions(filename), ByteBuffer.wrap(bytes)); + public void createFromBytes(BlobId blobId, byte[] bytes) throws StorageException { + createFromBytes(createBlobInfo(blobId), bytes); + } + + /** Creates a GCS file with the given byte contents and metadata, overwriting existing files. */ + public void createFromBytes(BlobInfo blobInfo, byte[] bytes) throws StorageException { + storage().create(blobInfo, bytes); + } + + /** Read the content of the given GCS file and return it in a byte array. */ + public byte[] readBytesFrom(BlobId blobId) throws StorageException { + return storage().readAllBytes(blobId); + } + + /** Delete the given GCS file. */ + public void delete(BlobId blobId) throws StorageException { + storage().delete(blobId); } /** @@ -86,37 +130,55 @@ public class GcsUtils { */ public ImmutableList listFolderObjects(String bucketName, String prefix) throws IOException { - ListResult result = - gcsService.list(bucketName, new ListOptions.Builder().setPrefix(prefix).build()); - final ImmutableList.Builder builder = new ImmutableList.Builder<>(); - result.forEachRemaining( - listItem -> { - if (!listItem.isDirectory()) { - builder.add(listItem.getName().replaceFirst(prefix, "")); - } - }); - return builder.build(); + return Streams.stream(storage().list(bucketName, BlobListOption.prefix(prefix)).iterateAll()) + .map(blob -> blob.getName().substring(prefix.length())) + .collect(toImmutableList()); } /** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */ - public boolean existsAndNotEmpty(GcsFilename file) { - GcsFileMetadata metadata; + public boolean existsAndNotEmpty(BlobId blobId) { try { - metadata = gcsService.getMetadata(file); - } catch (IOException e) { + Blob blob = storage().get(blobId); + return blob != null && blob.getSize() > 0; + } catch (StorageException e) { logger.atWarning().withCause(e).log("Failed to check if GCS file exists"); return false; } - return metadata != null && metadata.getLength() > 0; } - /** Determines most appropriate {@link GcsFileOptions} based on filename extension. */ - private static GcsFileOptions getOptions(GcsFilename filename) { - GcsFileOptions.Builder builder = new GcsFileOptions.Builder().cacheControl("no-cache"); - MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(filename.getObjectName()))); + /** Returns the user defined metadata of a GCS file if the file exists, or an empty map. */ + public ImmutableMap getMetadata(BlobId blobId) throws StorageException { + Blob blob = storage().get(blobId); + return blob == null ? ImmutableMap.of() : ImmutableMap.copyOf(blob.getMetadata()); + } + + /** + * Returns the {@link BlobInfo} of the given GCS file. + * + *

Note that a {@link Blob} is returned, but on the {@link BlobInfo} part of it is usable. + */ + public BlobInfo getBlobInfo(BlobId blobId) throws StorageException { + return storage().get(blobId); + } + + /** Determines most appropriate {@link BlobInfo} based on filename extension. */ + private static BlobInfo createBlobInfo(BlobId blobId) { + BlobInfo.Builder builder = BlobInfo.newBuilder(blobId).setCacheControl("no-cache"); + MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(blobId.getName()))); if (mediaType != null) { - builder = builder.mimeType(mediaType.type()); + builder = builder.setContentType(mediaType.toString()); } return builder.build(); } + + // These two methods are needed to check whether serialization is done correctly in tests. + @Override + public boolean equals(Object obj) { + return obj instanceof GcsUtils && ((GcsUtils) obj).storageOptions.equals(storageOptions); + } + + @Override + public int hashCode() { + return storageOptions.hashCode(); + } } diff --git a/core/src/main/java/google/registry/module/backend/BackendComponent.java b/core/src/main/java/google/registry/module/backend/BackendComponent.java index 17f0b9153..3ac38490a 100644 --- a/core/src/main/java/google/registry/module/backend/BackendComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendComponent.java @@ -26,7 +26,6 @@ import google.registry.export.datastore.DatastoreAdminModule; import google.registry.export.sheet.SheetsServiceModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; -import google.registry.gcs.GcsServiceModule; import google.registry.groups.DirectoryModule; import google.registry.groups.GroupsModule; import google.registry.groups.GroupssettingsModule; @@ -64,7 +63,6 @@ import javax.inject.Singleton; DirectoryModule.class, DummyKeyringModule.class, DriveModule.class, - GcsServiceModule.class, GroupsModule.class, GroupssettingsModule.class, JSchModule.class, diff --git a/core/src/main/java/google/registry/module/tools/ToolsComponent.java b/core/src/main/java/google/registry/module/tools/ToolsComponent.java index bfa012ef0..f1a6f806c 100644 --- a/core/src/main/java/google/registry/module/tools/ToolsComponent.java +++ b/core/src/main/java/google/registry/module/tools/ToolsComponent.java @@ -22,7 +22,6 @@ import google.registry.config.RegistryConfig.ConfigModule; import google.registry.export.DriveModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; -import google.registry.gcs.GcsServiceModule; import google.registry.groups.DirectoryModule; import google.registry.groups.GroupsModule; import google.registry.groups.GroupssettingsModule; @@ -54,7 +53,6 @@ import javax.inject.Singleton; DirectoryModule.class, DummyKeyringModule.class, DriveModule.class, - GcsServiceModule.class, GroupsModule.class, GroupssettingsModule.class, Jackson2Module.class, diff --git a/core/src/main/java/google/registry/rde/BrdaCopyAction.java b/core/src/main/java/google/registry/rde/BrdaCopyAction.java index 75a57cd15..fbe6777c8 100644 --- a/core/src/main/java/google/registry/rde/BrdaCopyAction.java +++ b/core/src/main/java/google/registry/rde/BrdaCopyAction.java @@ -17,7 +17,7 @@ package google.registry.rde; import static google.registry.model.rde.RdeMode.THIN; import static google.registry.request.Action.Method.POST; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.flogger.FluentLogger; import com.google.common.io.ByteStreams; import google.registry.config.RegistryConfig.Config; @@ -85,10 +85,10 @@ public final class BrdaCopyAction implements Runnable { private void copyAsRyde() throws IOException { String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0); - GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde"); - GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length"); - GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde"); - GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig"); + BlobId xmlFilename = BlobId.of(stagingBucket, prefix + ".xml.ghostryde"); + BlobId xmlLengthFilename = BlobId.of(stagingBucket, prefix + ".xml.length"); + BlobId rydeFile = BlobId.of(brdaBucket, prefix + ".ryde"); + BlobId sigFile = BlobId.of(brdaBucket, prefix + ".sig"); long xmlLength = readXmlLength(xmlLengthFilename); @@ -107,7 +107,7 @@ public final class BrdaCopyAction implements Runnable { } /** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */ - private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException { + private long readXmlLength(BlobId xmlLengthFilename) throws IOException { try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) { return Ghostryde.readLength(input); } diff --git a/core/src/main/java/google/registry/rde/RdeReportAction.java b/core/src/main/java/google/registry/rde/RdeReportAction.java index 72112a71b..2897d341f 100644 --- a/core/src/main/java/google/registry/rde/RdeReportAction.java +++ b/core/src/main/java/google/registry/rde/RdeReportAction.java @@ -23,7 +23,7 @@ import static google.registry.persistence.transaction.TransactionManagerUtil.tra import static google.registry.request.Action.Method.POST; import static google.registry.util.DateTimeUtils.isBeforeOrAt; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.flogger.FluentLogger; import com.google.common.io.ByteStreams; import google.registry.config.RegistryConfig.Config; @@ -97,7 +97,7 @@ public final class RdeReportAction implements Runnable, EscrowTask { .orElseThrow( () -> new IllegalStateException("RdeRevision was not set on generated deposit")); String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision); - GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde"); + BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde"); verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename); reporter.send(readReportFromGcs(reportFilename)); response.setContentType(PLAIN_TEXT_UTF_8); @@ -106,7 +106,7 @@ public final class RdeReportAction implements Runnable, EscrowTask { } /** Reads and decrypts the XML file from cloud storage. */ - private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException { + private byte[] readReportFromGcs(BlobId reportFilename) throws IOException { try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename); InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey)) { return ByteStreams.toByteArray(ghostrydeDecoder); diff --git a/core/src/main/java/google/registry/rde/RdeStagingAction.java b/core/src/main/java/google/registry/rde/RdeStagingAction.java index c3ac547a7..af75770dd 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingAction.java +++ b/core/src/main/java/google/registry/rde/RdeStagingAction.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Multimaps; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; import google.registry.mapreduce.MapreduceRunner; import google.registry.mapreduce.inputs.EppResourceInputs; import google.registry.mapreduce.inputs.NullInput; @@ -206,6 +207,7 @@ public final class RdeStagingAction implements Runnable { @Inject PendingDepositChecker pendingDepositChecker; @Inject RdeStagingReducer.Factory reducerFactory; @Inject Response response; + @Inject GcsUtils gcsUtils; @Inject MapreduceRunner mrRunner; @Inject @Config("transactionCooldown") Duration transactionCooldown; @Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual; @@ -234,7 +236,7 @@ public final class RdeStagingAction implements Runnable { } ValidationMode validationMode = lenient ? LENIENT : STRICT; RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings); - RdeStagingReducer reducer = reducerFactory.create(validationMode); + RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils); mrRunner .setJobName("Stage escrow deposits for all TLDs") diff --git a/core/src/main/java/google/registry/rde/RdeStagingReducer.java b/core/src/main/java/google/registry/rde/RdeStagingReducer.java index a6c0db649..f62c9ac46 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingReducer.java +++ b/core/src/main/java/google/registry/rde/RdeStagingReducer.java @@ -16,7 +16,6 @@ package google.registry.rde; import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime; @@ -24,10 +23,9 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.RetryParams; import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; @@ -69,27 +67,27 @@ public final class RdeStagingReducer extends Reducer new IllegalStateException("RdeRevision was not set on generated deposit")); final String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision); - final GcsFilename xmlFilename = new GcsFilename(bucket, name + ".xml.ghostryde"); - final GcsFilename xmlLengthFilename = new GcsFilename(bucket, name + ".xml.length"); - GcsFilename reportFilename = new GcsFilename(bucket, name + "-report.xml.ghostryde"); + final BlobId xmlFilename = BlobId.of(bucket, name + ".xml.ghostryde"); + final BlobId xmlLengthFilename = BlobId.of(bucket, name + ".xml.length"); + BlobId reportFilename = BlobId.of(bucket, name + "-report.xml.ghostryde"); verifyFileExists(xmlFilename); verifyFileExists(xmlLengthFilename); verifyFileExists(reportFilename); @@ -187,29 +187,30 @@ public final class RdeUploadAction implements Runnable, EscrowTask { } /** - * Performs a blocking upload of a cloud storage XML file to escrow provider, converting - * it to the RyDE format along the way by applying tar+compress+encrypt+sign, and saving the - * created RyDE file on GCS for future reference. + * Performs a blocking upload of a cloud storage XML file to escrow provider, converting it to the + * RyDE format along the way by applying tar+compress+encrypt+sign, and saving the created RyDE + * file on GCS for future reference. * *

This is done by layering a bunch of {@link java.io.FilterOutputStream FilterOutputStreams} - * on top of each other in reverse order that turn XML bytes into a RyDE file while - * simultaneously uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to - * blocking-copy bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline. + * on top of each other in reverse order that turn XML bytes into a RyDE file while simultaneously + * uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to blocking-copy + * bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline. * *

In pseudo-shell, the whole process looks like the following: * - * 

   {@code
-   *   gcs read $xmlFile \                                   # Get GhostRyDE from cloud storage.
-   *     | decrypt | decompress \                            # Convert it to XML.
-   *     | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
-   *     | tee gs://bucket/$rydeFilename.ryde \              # Save a copy of the RyDE file to GCS.
-   *     | sftp put $dstUrl/$rydeFilename.ryde \             # Upload to SFTP server.
-   *    && sftp put $dstUrl/$rydeFilename.sig  gs://bucket/$rydeFilename.sig      # Save a copy of signature to GCS.
-   *   }
+ * 
{@code
+   * gcs read $xmlFile \                                   # Get GhostRyDE from cloud storage.
+   *   | decrypt | decompress \                            # Convert it to XML.
+   *   | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
+   *   | tee gs://bucket/$rydeFilename.ryde \              # Save a copy of the RyDE file to GCS.
+   *   | sftp put $dstUrl/$rydeFilename.ryde \             # Upload to SFTP server.
+   *  && sftp put $dstUrl/$rydeFilename.sig  gs://bucket/$rydeFilename.sig      # Save a copy of signature to GCS.
+   *
+   * }
*/ @VisibleForTesting - protected void upload(GcsFilename xmlFile, long xmlLength, DateTime watermark, String name) + protected void upload(BlobId xmlFile, long xmlLength, DateTime watermark, String name) throws Exception { logger.atInfo().log("Uploading XML file '%s' to remote path '%s'.", xmlFile, uploadUrl); try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile); @@ -218,7 +219,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask { JSchSftpChannel ftpChan = session.openSftpChannel()) { ByteArrayOutputStream sigOut = new ByteArrayOutputStream(); String rydeFilename = name + ".ryde"; - GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename); + BlobId rydeGcsFilename = BlobId.of(bucket, rydeFilename); try (OutputStream ftpOutput = ftpChan.get().put(rydeFilename, sftpProgressMonitor, OVERWRITE); OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename); @@ -234,7 +235,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask { } String sigFilename = name + ".sig"; byte[] signature = sigOut.toByteArray(); - gcsUtils.createFromBytes(new GcsFilename(bucket, sigFilename), signature); + gcsUtils.createFromBytes(BlobId.of(bucket, sigFilename), signature); ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename); logger.atInfo().log("uploaded %,d bytes: %s", signature.length, sigFilename); } @@ -242,13 +243,13 @@ public final class RdeUploadAction implements Runnable, EscrowTask { } /** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */ - private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException { + private long readXmlLength(BlobId xmlLengthFilename) throws IOException { try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) { return Ghostryde.readLength(input); } } - private void verifyFileExists(GcsFilename filename) { + private void verifyFileExists(BlobId filename) { verify(gcsUtils.existsAndNotEmpty(filename), "Missing file: %s", filename); } } diff --git a/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java b/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java index f74fa9268..324a9315e 100644 --- a/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java +++ b/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java @@ -17,7 +17,7 @@ package google.registry.reporting.billing; import static com.google.common.base.Throwables.getRootCause; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.io.CharStreams; import com.google.common.net.MediaType; @@ -72,8 +72,7 @@ public class BillingEmailUtils { void emailOverallInvoice() { try { String invoiceFile = String.format("%s-%s.csv", invoiceFilePrefix, yearMonth); - GcsFilename invoiceFilename = - new GcsFilename(billingBucket, invoiceDirectoryPrefix + invoiceFile); + BlobId invoiceFilename = BlobId.of(billingBucket, invoiceDirectoryPrefix + invoiceFile); try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) { emailService.sendEmail( EmailMessage.newBuilder() diff --git a/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java b/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java index 8da9be513..b885fdf17 100644 --- a/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java +++ b/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java @@ -19,7 +19,7 @@ import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -120,7 +120,7 @@ public final class CopyDetailReportsAction implements Runnable { () -> { try (InputStream input = gcsUtils.openInputStream( - new GcsFilename(billingBucket, invoiceDirectoryPrefix + detailReportName))) { + BlobId.of(billingBucket, invoiceDirectoryPrefix + detailReportName))) { driveConnection.createOrUpdateFile( detailReportName, MediaType.CSV_UTF_8, diff --git a/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java b/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java index ef1213fab..557d9b616 100644 --- a/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java +++ b/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java @@ -21,7 +21,7 @@ import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.base.Ascii; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableCollection; @@ -260,7 +260,7 @@ public class IcannReportingStager { Ascii.toLowerCase(reportType.toString()), DateTimeFormat.forPattern("yyyyMM").print(yearMonth)); String reportBucketname = String.format("%s/%s", reportingBucket, subdir); - final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename); + final BlobId gcsFilename = BlobId.of(reportBucketname, reportFilename); gcsUtils.createFromBytes(gcsFilename, reportBytes); logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename); return reportFilename; @@ -269,7 +269,7 @@ public class IcannReportingStager { /** Creates and stores a manifest file on GCS, indicating which reports were generated. */ void createAndUploadManifest(String subdir, ImmutableList filenames) throws IOException { String reportBucketname = String.format("%s/%s", reportingBucket, subdir); - final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME); + final BlobId gcsFilename = BlobId.of(reportBucketname, MANIFEST_FILE_NAME); StringBuilder manifestString = new StringBuilder(); filenames.forEach((filename) -> manifestString.append(filename).append("\n")); gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8)); diff --git a/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java b/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java index 6860e19b1..a6b1c2dc7 100644 --- a/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java +++ b/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java @@ -20,7 +20,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -147,7 +147,7 @@ public final class IcannReportingUploadAction implements Runnable { cursorTimeMinusMonth.getYear(), cursorTimeMinusMonth.getMonthOfYear()); String reportBucketname = String.format("%s/%s", reportingBucket, reportSubdir); String filename = getFileName(cursorType, cursorTime, tldStr); - final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename); + final BlobId gcsFilename = BlobId.of(reportBucketname, filename); logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname); // Check that the report exists try { @@ -298,18 +298,18 @@ public final class IcannReportingUploadAction implements Runnable { emailService.sendEmail(EmailMessage.create(subject, body, recipient, sender)); } - private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException { + private byte[] readBytesFromGcs(BlobId reportFilename) throws IOException { try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) { return ByteStreams.toByteArray(gcsInput); } } - private void verifyFileExists(GcsFilename gcsFilename) { + private void verifyFileExists(BlobId gcsFilename) { checkArgument( gcsUtils.existsAndNotEmpty(gcsFilename), "Object %s in bucket %s not found", - gcsFilename.getObjectName(), - gcsFilename.getBucketName()); + gcsFilename.getName(), + gcsFilename.getBucket()); } } diff --git a/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java b/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java index d3adb241c..57d9dcd79 100644 --- a/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java +++ b/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java @@ -17,7 +17,7 @@ package google.registry.reporting.spec11; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -62,7 +62,7 @@ public class Spec11RegistrarThreatMatchesParser { } /** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */ - public ImmutableSet getFromFile(GcsFilename spec11ReportFilename) + public ImmutableSet getFromFile(BlobId spec11ReportFilename) throws IOException { if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) { return ImmutableSet.of(); @@ -81,7 +81,7 @@ public class Spec11RegistrarThreatMatchesParser { public Optional getPreviousDateWithMatches(LocalDate date) { LocalDate yesterday = date.minusDays(1); - GcsFilename gcsFilename = getGcsFilename(yesterday); + BlobId gcsFilename = getGcsFilename(yesterday); if (gcsUtils.existsAndNotEmpty(gcsFilename)) { return Optional.of(yesterday); } @@ -98,8 +98,8 @@ public class Spec11RegistrarThreatMatchesParser { return Optional.empty(); } - private GcsFilename getGcsFilename(LocalDate localDate) { - return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate)); + private BlobId getGcsFilename(LocalDate localDate) { + return BlobId.of(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate)); } private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException { diff --git a/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java b/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java index 947b0846f..0fadac0bb 100644 --- a/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java +++ b/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java @@ -14,7 +14,6 @@ package google.registry.tools.server; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterators.filter; import static com.google.common.io.BaseEncoding.base16; @@ -25,11 +24,10 @@ import static google.registry.request.Action.Method.POST; import static java.nio.charset.StandardCharsets.UTF_8; import static org.joda.time.DateTimeZone.UTC; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.RetryParams; import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -98,13 +96,14 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA @Inject MapreduceRunner mrRunner; @Inject JsonActionRunner jsonActionRunner; @Inject @Config("zoneFilesBucket") String bucket; - @Inject @Config("gcsBufferSize") int gcsBufferSize; @Inject @Config("commitLogDatastoreRetention") Duration datastoreRetention; @Inject @Config("dnsDefaultATtl") Duration dnsDefaultATtl; @SuppressWarnings("DurationVariableWithUnits") // false-positive Error Prone check @Inject @Config("dnsDefaultNsTtl") Duration dnsDefaultNsTtl; @Inject @Config("dnsDefaultDsTtl") Duration dnsDefaultDsTtl; @Inject Clock clock; + @Inject GcsUtils gcsUtils; + @Inject GenerateZoneFilesAction() {} @Override @@ -140,7 +139,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA .runMapreduce( new GenerateBindFileMapper( tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl), - new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize), + new GenerateBindFileReducer(bucket, exportTime, gcsUtils), ImmutableList.of(new NullInput<>(), createEntityInput(DomainBase.class))) .getLinkToMapreduceConsole(); ImmutableList filenames = @@ -236,22 +235,19 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA private final String bucket; private final DateTime exportTime; - private final int gcsBufferSize; + private final GcsUtils gcsUtils; - GenerateBindFileReducer(String bucket, DateTime exportTime, int gcsBufferSize) { + GenerateBindFileReducer(String bucket, DateTime exportTime, GcsUtils gcsUtils) { this.bucket = bucket; this.exportTime = exportTime; - this.gcsBufferSize = gcsBufferSize; + this.gcsUtils = gcsUtils; } @Override public void reduce(String tld, ReducerInput stanzas) { String stanzaCounter = tld + " stanzas"; - GcsFilename filename = - new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime)); - GcsUtils cloudStorage = - new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize); - try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename); + BlobId filename = BlobId.of(bucket, String.format(FILENAME_FORMAT, tld, exportTime)); + try (OutputStream gcsOutput = gcsUtils.openOutputStream(filename); Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8); PrintWriter writer = new PrintWriter(osWriter)) { writer.printf(HEADER_FORMAT, tld); diff --git a/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java b/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java index d75b54fe2..60d6c3761 100644 --- a/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java +++ b/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java @@ -24,18 +24,17 @@ import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.util.DateTimeUtils.START_OF_TIME; import static org.joda.time.DateTimeZone.UTC; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableMap; import com.googlecode.objectify.Key; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.ImmutableObject; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; import google.registry.testing.AppEngineExtension; -import google.registry.testing.GcsTestingUtils; import google.registry.testing.TestObject; import java.util.List; import org.joda.time.DateTime; @@ -53,8 +52,7 @@ public class ExportCommitLogDiffActionTest { .withOfyTestEntities(TestObject.class) .build(); - /** Local GCS service available for testing. */ - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final DateTime now = DateTime.now(UTC); private final DateTime oneMinuteAgo = now.minusMinutes(1); @@ -63,7 +61,7 @@ public class ExportCommitLogDiffActionTest { @BeforeEach void beforeEach() { - task.gcsService = gcsService; + task.gcsUtils = gcsUtils; task.gcsBucket = "gcs bucket"; task.batchSize = 5; } @@ -84,10 +82,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -95,8 +94,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "0"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported).containsExactly(upperCheckpoint); } @@ -139,10 +137,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -150,8 +149,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "4"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in time order, with matching mutations. CommitLogManifest manifest1 = createManifest(2, now.minusDays(1).minusMillis(1)); @@ -191,10 +189,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -202,8 +201,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "4"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in the order below, with matching mutations. CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo); @@ -246,10 +244,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -257,8 +256,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "6"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in the order below, with matching mutations. CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo); @@ -301,10 +299,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -312,8 +311,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "0"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); // We expect no manifests or mutations, only the upper checkpoint. assertThat(exported).containsExactly(upperCheckpoint); } @@ -359,11 +357,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)) - .isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -371,8 +369,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "6"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in time order, with matching mutations. CommitLogManifest manifest1 = createManifest(3, oneMinuteAgo.minusDays(2)); @@ -415,10 +412,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, START_OF_TIME.toString(), @@ -426,8 +424,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "3"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in the order below, with matching mutations. CommitLogManifest manifest1 = createManifest(1, START_OF_TIME.plusMillis(1)); diff --git a/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java b/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java index 7d4dd5de0..ff9a67d69 100644 --- a/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java +++ b/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java @@ -19,29 +19,24 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT; import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX; -import static java.lang.reflect.Proxy.newProxyInstance; import static org.joda.time.DateTimeZone.UTC; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; -import com.google.appengine.tools.cloudstorage.ListItem; -import com.google.appengine.tools.cloudstorage.ListResult; -import com.google.common.collect.Iterators; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.flogger.LoggerConfig; import com.google.common.testing.TestLogHandler; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.testing.AppEngineExtension; import java.io.IOException; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.Callable; import java.util.logging.LogRecord; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; @@ -55,7 +50,7 @@ public class GcsDiffFileListerTest { private final DateTime now = DateTime.now(UTC); private final GcsDiffFileLister diffLister = new GcsDiffFileLister(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final TestLogHandler logHandler = new TestLogHandler(); @RegisterExtension @@ -64,25 +59,18 @@ public class GcsDiffFileListerTest { @BeforeEach void beforeEach() throws Exception { - diffLister.gcsService = gcsService; + diffLister.gcsUtils = gcsUtils; diffLister.executor = newDirectExecutorService(); for (int i = 0; i < 5; i++) { - gcsService.createOrReplace( - new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(i)), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(i + 1).toString()) - .build(), - ByteBuffer.wrap(new byte[]{1, 2, 3})); + addGcsFile(i, i + 1); } LoggerConfig.getConfig(GcsDiffFileLister.class).addHandler(logHandler); } - private Iterable extractTimesFromDiffFiles(List diffFiles) { + private Iterable extractTimesFromDiffFiles(ImmutableList diffFiles) { return transform( diffFiles, - metadata -> - DateTime.parse( - metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()))); + blobInfo -> DateTime.parse(blobInfo.getName().substring(DIFF_FILE_PREFIX.length()))); } private Iterable listDiffFiles(DateTime fromTime, DateTime toTime) { @@ -90,12 +78,12 @@ public class GcsDiffFileListerTest { } private void addGcsFile(int fileAge, int prevAge) throws IOException { - gcsService.createOrReplace( - new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString()) - .build(), - ByteBuffer.wrap(new byte[]{1, 2, 3})); + BlobInfo blobInfo = + BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge))) + .setMetadata( + ImmutableMap.of(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString())) + .build(); + gcsUtils.createFromBytes(blobInfo, new byte[] {1, 2, 3}); } private void assertLogContains(String message) { @@ -114,38 +102,11 @@ public class GcsDiffFileListerTest { } @Test - void testList_patchesHoles() { - // Fake out the GCS list() method to return only the first and last file. - // We can't use Mockito.spy() because GcsService's impl is final. - diffLister.gcsService = (GcsService) newProxyInstance( - GcsService.class.getClassLoader(), - new Class[] {GcsService.class}, - new InvocationHandler() { - @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - if (method.getName().equals("list")) { - // ListResult is an incredibly annoying thing to construct. It needs to be fed from a - // Callable that returns Iterators, each representing a batch of results. - return new ListResult(new Callable>() { - boolean called = false; - - @Override - public Iterator call() { - try { - return called ? null : Iterators.forArray( - new ListItem.Builder() - .setName(DIFF_FILE_PREFIX + now) - .build(), - new ListItem.Builder() - .setName(DIFF_FILE_PREFIX + now.minusMinutes(4)) - .build()); - } finally { - called = true; - } - }}); - } - return method.invoke(gcsService, args); - }}); + void testList_patchesHoles() throws Exception { + GcsUtils mockGcsUtils = mock(GcsUtils.class); + diffLister.gcsUtils = spy(gcsUtils); + when(mockGcsUtils.listFolderObjects(anyString(), anyString())) + .thenReturn(ImmutableList.of(now.toString(), now.minusMinutes(4).toString())); DateTime fromTime = now.minusMinutes(4).minusSeconds(1); // Request all files with checkpoint > fromTime. assertThat(listDiffFiles(fromTime, null)) diff --git a/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java b/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java index 693f015ac..40c7a2db1 100644 --- a/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java +++ b/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java @@ -37,14 +37,14 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import com.google.common.truth.Truth8; import com.googlecode.objectify.Key; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.common.DatabaseMigrationStateSchedule; import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; import google.registry.model.contact.ContactResource; @@ -110,7 +110,7 @@ public class ReplayCommitLogsToSqlActionTest { .build(); /** Local GCS service. */ - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final ReplayCommitLogsToSqlAction action = new ReplayCommitLogsToSqlAction(); private final FakeResponse response = new FakeResponse(); @@ -123,13 +123,13 @@ public class ReplayCommitLogsToSqlActionTest { @BeforeEach void beforeEach() { - action.gcsService = gcsService; + action.gcsUtils = gcsUtils; action.response = response; action.requestStatusChecker = requestStatusChecker; action.clock = fakeClock; action.gcsBucket = "gcs bucket"; action.diffLister = new GcsDiffFileLister(); - action.diffLister.gcsService = gcsService; + action.diffLister.gcsUtils = gcsUtils; action.diffLister.executor = newDirectExecutorService(); ofyTm() .transact( @@ -165,9 +165,9 @@ public class ReplayCommitLogsToSqlActionTest { CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2)); Key manifest2Key = CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1)); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -182,7 +182,7 @@ public class ReplayCommitLogsToSqlActionTest { CommitLogMutation.create(manifest1bKey, TestObject.create("c")), CommitLogMutation.create(manifest1bKey, TestObject.create("d"))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -200,8 +200,8 @@ public class ReplayCommitLogsToSqlActionTest { void testReplay_noManifests() throws Exception { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep"))); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); - saveDiffFile(gcsService, createCheckpoint(now.minusMillis(2))); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); + saveDiffFile(gcsUtils, createCheckpoint(now.minusMillis(2))); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1))); runAndAssertSuccess(now.minusMillis(1)); assertExpectedIds("previous to keep"); @@ -214,10 +214,10 @@ public class ReplayCommitLogsToSqlActionTest { jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep"))); Key bucketKey = getBucketKey(1); Key manifestKey = CommitLogManifest.createKey(bucketKey, now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a")), @@ -237,10 +237,10 @@ public class ReplayCommitLogsToSqlActionTest { jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep"))); Key bucketKey = getBucketKey(1); Key manifestKey = CommitLogManifest.createKey(bucketKey, now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a")), @@ -258,10 +258,10 @@ public class ReplayCommitLogsToSqlActionTest { jpaTm().insertWithoutBackup(TestObject.create("previous to keep")); jpaTm().insertWithoutBackup(TestObject.create("previous to delete")); }); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -276,10 +276,10 @@ public class ReplayCommitLogsToSqlActionTest { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> jpaTm().put(TestObject.create("existing", "a"))); Key manifestKey = CommitLogManifest.createKey(getBucketKey(1), now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMillis(1)), CommitLogManifest.create(getBucketKey(1), now, null), CommitLogMutation.create(manifestKey, TestObject.create("existing", "b"))); @@ -294,10 +294,10 @@ public class ReplayCommitLogsToSqlActionTest { void testReplay_deleteMissingEntity() throws Exception { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> jpaTm().put(TestObject.create("previous to keep", "a"))); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMillis(1)), CommitLogManifest.create( getBucketKey(1), @@ -332,7 +332,7 @@ public class ReplayCommitLogsToSqlActionTest { TransactionManagerFactory.setJpaTm(() -> spy); // Save in the commit logs the domain and contact (in that order) and the token deletion saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(toDelete))), @@ -375,7 +375,7 @@ public class ReplayCommitLogsToSqlActionTest { TransactionManagerFactory.setJpaTm(() -> spy); // Save two commits -- the deletion, then the new version of the contact saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1).plusMillis(1)), CommitLogManifest.create( getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(contact))), @@ -415,7 +415,7 @@ public class ReplayCommitLogsToSqlActionTest { () -> { try { saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), now.minusMinutes(1), ImmutableSet.of()), @@ -444,7 +444,7 @@ public class ReplayCommitLogsToSqlActionTest { // Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant) Key claimsListKey = Key.create(ClaimsList.class, 1L); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -491,7 +491,7 @@ public class ReplayCommitLogsToSqlActionTest { Key manifestKey = CommitLogManifest.createKey(bucketKey, now); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a"))); @@ -504,7 +504,7 @@ public class ReplayCommitLogsToSqlActionTest { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), diff --git a/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java b/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java index 9fd9c3522..2b72b060c 100644 --- a/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java +++ b/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java @@ -14,7 +14,6 @@ package google.registry.backup; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Maps.toMap; import static com.google.common.truth.Truth.assertThat; @@ -28,15 +27,16 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy; import static org.joda.time.DateTimeZone.UTC; import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.io.Resources; import com.google.common.primitives.Longs; import com.googlecode.objectify.Key; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.ImmutableObject; import google.registry.model.domain.DomainBase; import google.registry.model.ofy.CommitLogBucket; @@ -51,7 +51,6 @@ import google.registry.testing.TestObject; import google.registry.util.Retrier; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -68,7 +67,7 @@ public class RestoreCommitLogsActionTest { private final DateTime now = DateTime.now(UTC); private final RestoreCommitLogsAction action = new RestoreCommitLogsAction(); - private final GcsService gcsService = createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @RegisterExtension public final AppEngineExtension appEngine = @@ -79,7 +78,7 @@ public class RestoreCommitLogsActionTest { @BeforeEach void beforeEach() { - action.gcsService = gcsService; + action.gcsUtils = gcsUtils; action.dryRun = false; action.datastoreService = DatastoreServiceFactory.getDatastoreService(); action.fromTime = now.minusMillis(1); @@ -87,7 +86,7 @@ public class RestoreCommitLogsActionTest { action.defaultGcsBucket = GCS_BUCKET; action.gcsBucketOverride = Optional.empty(); action.diffLister = new GcsDiffFileLister(); - action.diffLister.gcsService = gcsService; + action.diffLister.gcsUtils = gcsUtils; action.diffLister.executor = newDirectExecutorService(); } @@ -109,10 +108,10 @@ public class RestoreCommitLogsActionTest { CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2)); Key manifest2Key = CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1)); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); Iterable file1CommitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -128,7 +127,7 @@ public class RestoreCommitLogsActionTest { CommitLogMutation.create(manifest1bKey, TestObject.create("d"))); Iterable file2CommitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -149,8 +148,8 @@ public class RestoreCommitLogsActionTest { @Test void testRestore_noManifests() throws Exception { auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); - Iterable commitLogs = saveDiffFile(gcsService, createCheckpoint(now)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); + Iterable commitLogs = saveDiffFile(gcsUtils, createCheckpoint(now)); action.run(); auditedOfy().clearSessionCache(); assertExpectedIds("previous to keep"); @@ -164,10 +163,10 @@ public class RestoreCommitLogsActionTest { auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now(); Key bucketKey = getBucketKey(1); Key manifestKey = CommitLogManifest.createKey(bucketKey, now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a")), @@ -186,10 +185,10 @@ public class RestoreCommitLogsActionTest { .saveWithoutBackup() .entities(TestObject.create("previous to keep"), TestObject.create("previous to delete")) .now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -207,12 +206,10 @@ public class RestoreCommitLogsActionTest { @Test void testRestore_manifestWithNoMutationsOrDeletions() throws Exception { auditedOfy().saveWithoutBackup().entities(TestObject.create("previous to keep")).now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, - createCheckpoint(now), - CommitLogManifest.create(getBucketKey(1), now, null)); + gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null)); action.run(); auditedOfy().clearSessionCache(); assertExpectedIds("previous to keep"); @@ -225,10 +222,10 @@ public class RestoreCommitLogsActionTest { void testRestore_mutateExistingEntity() throws Exception { auditedOfy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now(); Key manifestKey = CommitLogManifest.createKey(getBucketKey(1), now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null), CommitLogMutation.create(manifestKey, TestObject.create("existing", "b"))); @@ -245,10 +242,10 @@ public class RestoreCommitLogsActionTest { @Test void testRestore_deleteMissingEntity() throws Exception { auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -270,7 +267,7 @@ public class RestoreCommitLogsActionTest { // imported, in particular, the domain's 'registrant' key can be used by Objectify to load the // contact. saveDiffFile( - gcsService, + gcsUtils, Resources.toByteArray(Resources.getResource("google/registry/backup/commitlog.data")), now); action.run(); @@ -286,18 +283,18 @@ public class RestoreCommitLogsActionTest { return CommitLogCheckpoint.create(now, toMap(getBucketIds(), x -> now)); } - static void saveDiffFile(GcsService gcsService, byte[] rawBytes, DateTime timestamp) + static void saveDiffFile(GcsUtils gcsUtils, byte[] rawBytes, DateTime timestamp) throws IOException { - gcsService.createOrReplace( - new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString()) - .build(), - ByteBuffer.wrap(rawBytes)); + BlobInfo blobInfo = + BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp)) + .setMetadata( + ImmutableMap.of(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString())) + .build(); + gcsUtils.createFromBytes(blobInfo, rawBytes); } static Iterable saveDiffFile( - GcsService gcsService, CommitLogCheckpoint checkpoint, ImmutableObject... entities) + GcsUtils gcsUtils, CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException { DateTime now = checkpoint.getCheckpointTime(); List allEntities = Lists.asList(checkpoint, entities); @@ -305,13 +302,13 @@ public class RestoreCommitLogsActionTest { for (ImmutableObject entity : allEntities) { serializeEntity(entity, output); } - saveDiffFile(gcsService, output.toByteArray(), now); + saveDiffFile(gcsUtils, output.toByteArray(), now); return allEntities; } - static void saveDiffFileNotToRestore(GcsService gcsService, DateTime now) throws Exception { + static void saveDiffFileNotToRestore(GcsUtils gcsUtils, DateTime now) throws Exception { saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null), CommitLogMutation.create( diff --git a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java index 431291ee1..2af29c6c9 100644 --- a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java +++ b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java @@ -14,14 +14,12 @@ package google.registry.export; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.truth.Truth.assertThat; import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_FILENAME; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistDeletedDomain; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.eq; @@ -29,12 +27,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.ListOptions; -import com.google.appengine.tools.cloudstorage.ListResult; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.StorageException; +import com.google.common.collect.ImmutableList; import com.google.common.net.MediaType; import google.registry.export.ExportDomainListsAction.ExportDomainListsReducer; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.ofy.Ofy; import google.registry.model.registry.Registry; import google.registry.model.registry.Registry.TldType; @@ -46,7 +45,6 @@ import google.registry.testing.InjectExtension; import google.registry.testing.TestOfyAndSql; import google.registry.testing.TestOfyOnly; import google.registry.testing.mapreduce.MapreduceTestCase; -import java.io.FileNotFoundException; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; @@ -57,7 +55,7 @@ import org.mockito.ArgumentCaptor; @DualDatabaseTest class ExportDomainListsActionTest extends MapreduceTestCase { - private GcsService gcsService; + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private DriveConnection driveConnection = mock(DriveConnection.class); private ArgumentCaptor bytesExportedToDrive = ArgumentCaptor.forClass(byte[].class); private final FakeResponse response = new FakeResponse(); @@ -81,10 +79,9 @@ class ExportDomainListsActionTest extends MapreduceTestCase readGcsFile(gcsService, nonexistentFile)); - ListResult ls = gcsService.list("outputbucket", ListOptions.DEFAULT); - assertThat(ls.next().getName()).isEqualTo("tld.txt"); - // Make sure that no other files were written out. - assertThat(ls.hasNext()).isFalse(); + BlobId nonexistentFile = BlobId.of("outputbucket", "testtld.txt"); + assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile)); + ImmutableList ls = gcsUtils.listFolderObjects("outputbucket", ""); + assertThat(ls).containsExactly("tld.txt"); verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld"); verifyNoMoreInteractions(driveConnection); } @@ -160,14 +155,14 @@ class ExportDomainListsActionTest extends MapreduceTestCase metadata = ImmutableMap.of("key1", "val1", "Key2", "val2"); + private final byte[] bytes = new byte[] {'a', 'b', 'c'}; + + @BeforeEach + void beforeEach() {} + + @Test + void testSerialization_testStorage() throws Exception { + assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils); + } + + @Test + void testSerialization_realStorage() throws Exception { + gcsUtils = new GcsUtils(StorageOptions.getDefaultInstance()); + assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils); + } + + @Test + void testStreams() throws Exception { + try (OutputStream os = gcsUtils.openOutputStream(blobId, metadata)) { + os.write(bytes); + os.flush(); + } + assertThat(ByteStreams.toByteArray(gcsUtils.openInputStream(blobId))).isEqualTo(bytes); + assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).containsExactlyEntriesIn(metadata); + } + + @Test + void testCreateListReadDelete() throws Exception { + gcsUtils.createFromBytes(BlobInfo.newBuilder(blobId).setMetadata(metadata).build(), bytes); + assertThat(gcsUtils.existsAndNotEmpty(blobId)).isTrue(); + assertThat(gcsUtils.listFolderObjects(bucket, "")).containsExactly("my-file"); + assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).isEqualTo(metadata); + assertThat(gcsUtils.readBytesFrom(blobId)).isEqualTo(bytes); + gcsUtils.delete(blobId); + assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse(); + assertThat(gcsUtils.listFolderObjects(bucket, "")).isEmpty(); + } + + @Test + void testSetContentType() { + blobId = BlobId.of(bucket, "something.json"); + gcsUtils.createFromBytes(blobId, bytes); + assertThat(gcsUtils.getBlobInfo(blobId).getContentType()) + .isEqualTo(MediaType.JSON_UTF_8.toString()); + } + + @Test + void testList() throws Exception { + ImmutableList blobIds = + ImmutableList.of( + BlobId.of(bucket, "a/b/xyz.txt"), + BlobId.of(bucket, "a/cde.exe"), + BlobId.of(bucket, "fgh.jpg"), + BlobId.of(bucket, "c/ijk.mp4")); + + for (BlobId blobId : blobIds) { + gcsUtils.createFromBytes(blobId, bytes); + } + assertThat(gcsUtils.listFolderObjects(bucket, "")) + .containsExactly("a/b/xyz.txt", "a/cde.exe", "fgh.jpg", "c/ijk.mp4"); + assertThat(gcsUtils.listFolderObjects(bucket, "a/")).containsExactly("b/xyz.txt", "cde.exe"); + } + + @Test + void testEmptyFile() { + gcsUtils.createFromBytes(blobId, new byte[] {}); + assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse(); + } + + private static byte[] serialize(Object object) throws IOException { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(object); + oos.flush(); + return baos.toByteArray(); + } + } + + private static Object deserialize(byte[] bytes) throws IOException, ClassNotFoundException { + try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) { + return ois.readObject(); + } + } +} diff --git a/core/src/test/java/google/registry/gcs/backport/FakeStorageRpc.java b/core/src/test/java/google/registry/gcs/backport/FakeStorageRpc.java new file mode 100644 index 000000000..74210e68e --- /dev/null +++ b/core/src/test/java/google/registry/gcs/backport/FakeStorageRpc.java @@ -0,0 +1,517 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package google.registry.gcs.backport; + +import com.google.api.client.util.DateTime; +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.ServiceAccount; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.Tuple; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.testing.StorageRpcTestBase; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.nio.file.FileAlreadyExistsException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * A bare-bones in-memory implementation of StorageRpc, meant for testing. + * + *

This class is not thread-safe. It's also (currently) limited in the following ways: + * + *

    + *
  • Supported + *
      + *
    • object create + *
    • object get + *
    • object delete + *
    • list the contents of a bucket + *
    • generations + *
    + *
  • Unsupported + *
      + *
    • bucket create + *
    • bucket get + *
    • bucket delete + *
    • list all buckets + *
    • file attributes + *
    • patch + *
    • continueRewrite + *
    • createBatch + *
    • checksums, etags + *
    • IAM operations + *
    • BucketLock operations + *
    • HMAC key operations + *
    + *
+ */ +@NotThreadSafe +class FakeStorageRpc extends StorageRpcTestBase { + + private static final SimpleDateFormat RFC_3339_FORMATTER = + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); + + // fullname -> metadata + Map metadata = new HashMap<>(); + // fullname -> contents + Map contents = new HashMap<>(); + // fullname -> future contents that will be visible on close. + Map futureContents = new HashMap<>(); + + private final boolean throwIfOption; + + /** @param throwIfOption if true, we throw when given any option */ + public FakeStorageRpc(boolean throwIfOption) { + this.throwIfOption = throwIfOption; + } + + // remove all files + void reset() { + metadata = new HashMap<>(); + contents = new HashMap<>(); + } + + @Override + public StorageObject create(StorageObject object, InputStream content, Map options) + throws StorageException { + potentiallyThrow(options); + String key = fullname(object); + object.setUpdated(now()); + metadata.put(key, object); + try { + contents.put(key, com.google.common.io.ByteStreams.toByteArray(content)); + } catch (IOException e) { + throw new StorageException(e); + } + // TODO: crc, etc + return object; + } + + @Override + public Tuple> list(String bucket, Map options) + throws StorageException { + String delimiter = null; + String preprefix = ""; + String pageToken = null; + long maxResults = Long.MAX_VALUE; + for (Map.Entry e : options.entrySet()) { + switch (e.getKey()) { + case PAGE_TOKEN: + pageToken = (String) e.getValue(); + break; + case PREFIX: + preprefix = (String) e.getValue(); + if (preprefix.startsWith("/")) { + preprefix = preprefix.substring(1); + } + break; + case DELIMITER: + delimiter = (String) e.getValue(); + break; + case FIELDS: + // ignore and return all the fields + break; + case MAX_RESULTS: + maxResults = (Long) e.getValue(); + break; + case USER_PROJECT: + // prevent unsupported operation + break; + default: + throw new UnsupportedOperationException("Unknown option: " + e.getKey()); + } + } + final String prefix = preprefix; + + List values = new ArrayList<>(); + Map folders = new HashMap<>(); + for (StorageObject so : metadata.values()) { + if (!so.getBucket().equals(bucket) || !so.getName().startsWith(prefix)) { + continue; + } + if (processedAsFolder(so, delimiter, prefix, folders)) { + continue; + } + so.setSize(size(so)); + values.add(so); + } + values.addAll(folders.values()); + + // truncate to max allowed length + if (values.size() > maxResults) { + List newValues = new ArrayList<>(); + for (int i = 0; i < maxResults; i++) { + newValues.add(values.get(i)); + } + values = newValues; + } + + // null cursor to indicate there is no more data (empty string would cause us to be called + // again). + // The type cast seems to be necessary to help Java's typesystem remember that collections are + // iterable. + return Tuple.of(pageToken, (Iterable) values); + } + + /** Returns the requested bucket or {@code null} if not found. */ + @Override + public Bucket get(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + /** Returns the requested storage object or {@code null} if not found. */ + @Override + public StorageObject get(StorageObject object, Map options) throws StorageException { + // we allow the "ID" option because we need to, but then we give a whole answer anyways + // because the caller won't mind the extra fields. + if (throwIfOption + && !options.isEmpty() + && options.size() > 1 + && options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) { + throw new UnsupportedOperationException(); + } + + String key = fullname(object); + if (metadata.containsKey(key)) { + StorageObject ret = metadata.get(key); + ret.setSize(size(ret)); + ret.setId(key); + + return ret; + } + return null; + } + + @Override + public Bucket patch(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public StorageObject patch(StorageObject storageObject, Map options) + throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public boolean delete(Bucket bucket, Map options) throws StorageException { + return false; + } + + @Override + public boolean delete(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + contents.remove(key); + return null != metadata.remove(key); + } + + @Override + public StorageObject compose( + Iterable sources, StorageObject target, Map targetOptions) + throws StorageException { + return null; + } + + @Override + public byte[] load(StorageObject storageObject, Map options) throws StorageException { + String key = fullname(storageObject); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + return contents.get(key); + } + + @Override + public Tuple read( + StorageObject from, Map options, long zposition, int zbytes) + throws StorageException { + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option op : options.keySet()) { + if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) { + generationMatch = (Long) options.get(op); + } else { + throw new UnsupportedOperationException("Unknown option: " + op); + } + } + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + checkGeneration(key, generationMatch); + long position = zposition; + int bytes = zbytes; + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + if ((int) position + bytes > full.length) { + bytes = full.length - (int) position; + } + if (bytes <= 0) { + // special case: you're trying to read past the end + return Tuple.of("etag-goes-here", new byte[0]); + } + byte[] ret = new byte[bytes]; + System.arraycopy(full, (int) position, ret, 0, bytes); + return Tuple.of("etag-goes-here", ret); + } + + @Override + public long read( + StorageObject from, Map options, long position, OutputStream outputStream) { + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option op : options.keySet()) { + if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) { + generationMatch = (Long) options.get(op); + } else { + throw new UnsupportedOperationException("Unknown option: " + op); + } + } + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + checkGeneration(key, generationMatch); + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + int bytes = (int) (full.length - position); + if (bytes <= 0) { + // special case: you're trying to read past the end + return 0; + } + try { + outputStream.write(full, (int) position, bytes); + } catch (IOException e) { + throw new StorageException(500, "Failed to write to file", e); + } + return bytes; + } + + @Override + public String open(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option option : options.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH) { + generationMatch = (Long) options.get(option); + } + } + checkGeneration(key, generationMatch); + metadata.put(key, object); + + return fullname(object); + } + + @Override + public String open(String signedURL) { + return null; + } + + @Override + public void write( + String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, int length, boolean last) + throws StorageException { + writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last); + } + + @Override + public StorageObject writeWithResponse( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last) { + // this may have a lot more allocations than ideal, but it'll work. + byte[] bytes; + if (futureContents.containsKey(uploadId)) { + bytes = futureContents.get(uploadId); + if (bytes.length < length + destOffset) { + byte[] newBytes = new byte[(int) (length + destOffset)]; + System.arraycopy(bytes, 0, newBytes, (int) 0, bytes.length); + bytes = newBytes; + } + } else { + bytes = new byte[(int) (length + destOffset)]; + } + System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length); + // we want to mimic the GCS behavior that file contents are only visible on close. + StorageObject storageObject = null; + if (last) { + contents.put(uploadId, bytes); + futureContents.remove(uploadId); + if (metadata.containsKey(uploadId)) { + storageObject = metadata.get(uploadId); + storageObject.setUpdated(now()); + Long generation = storageObject.getGeneration(); + if (null == generation) { + generation = Long.valueOf(0); + } + storageObject.setGeneration(++generation); + metadata.put(uploadId, storageObject); + } + } else { + futureContents.put(uploadId, bytes); + } + return storageObject; + } + + @Override + public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException { + String sourceKey = fullname(rewriteRequest.source); + + // a little hackish, just good enough for the tests to work. + if (!contents.containsKey(sourceKey)) { + throw new StorageException(404, "File not found: " + sourceKey); + } + + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option option : rewriteRequest.targetOptions.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH) { + generationMatch = (Long) rewriteRequest.targetOptions.get(option); + } + } + + String destKey = fullname(rewriteRequest.target); + + // if this is a new file, set generation to 1, else increment the existing generation + long generation = 1; + if (metadata.containsKey(destKey)) { + generation = metadata.get(destKey).getGeneration() + 1; + } + + checkGeneration(destKey, generationMatch); + + byte[] data = contents.get(sourceKey); + + rewriteRequest.target.setGeneration(generation); + rewriteRequest.target.setSize(BigInteger.valueOf(data.length)); + rewriteRequest.target.setUpdated(metadata.get(sourceKey).getUpdated()); + + metadata.put(destKey, rewriteRequest.target); + + contents.put(destKey, Arrays.copyOf(data, data.length)); + return new RewriteResponse( + rewriteRequest, + rewriteRequest.target, + data.length, + true, + "rewriteToken goes here", + data.length); + } + + private static DateTime now() { + return DateTime.parseRfc3339(RFC_3339_FORMATTER.format(new Date())); + } + + private String fullname(StorageObject so) { + return (so.getBucket() + "/" + so.getName()); + } + + private BigInteger size(StorageObject so) { + String key = fullname(so); + + if (contents.containsKey(key)) { + return BigInteger.valueOf(contents.get(key).length); + } + + return null; + } + + private void potentiallyThrow(Map options) throws UnsupportedOperationException { + if (throwIfOption && !options.isEmpty()) { + throw new UnsupportedOperationException(); + } + } + + /** + * Throw if we're asking for generation 0 and the file exists, or if the requested generation + * number doesn't match what is asked. + * + * @param key + * @param generationMatch + */ + private void checkGeneration(String key, Long generationMatch) { + if (null == generationMatch) { + return; + } + if (generationMatch == 0 && metadata.containsKey(key)) { + throw new StorageException(new FileAlreadyExistsException(key)); + } + if (generationMatch != 0) { + Long generation = metadata.get(key).getGeneration(); + if (!generationMatch.equals(generation)) { + throw new StorageException( + 404, "Generation mismatch. Requested " + generationMatch + " but got " + generation); + } + } + } + + // Returns true if this is a folder. Adds it to folders if it isn't already there. + private static boolean processedAsFolder( + StorageObject so, + String delimiter, + String prefix, /* inout */ + Map folders) { + if (delimiter == null) { + return false; + } + int nextSlash = so.getName().indexOf(delimiter, prefix.length()); + if (nextSlash < 0) { + return false; + } + String folderName = so.getName().substring(0, nextSlash + 1); + if (folders.containsKey(folderName)) { + return true; + } + StorageObject fakeFolder = new StorageObject(); + fakeFolder.setName(folderName); + fakeFolder.setBucket(so.getBucket()); + fakeFolder.setGeneration(so.getGeneration()); + fakeFolder.set("isDirectory", true); + fakeFolder.setSize(BigInteger.ZERO); + folders.put(folderName, fakeFolder); + return true; + } + + @Override + public ServiceAccount getServiceAccount(String projectId) { + return null; + } +} diff --git a/core/src/test/java/google/registry/gcs/backport/LocalStorageHelper.java b/core/src/test/java/google/registry/gcs/backport/LocalStorageHelper.java new file mode 100644 index 000000000..e4b51c348 --- /dev/null +++ b/core/src/test/java/google/registry/gcs/backport/LocalStorageHelper.java @@ -0,0 +1,106 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package google.registry.gcs.backport; + +import com.google.cloud.ServiceRpc; +import com.google.cloud.spi.ServiceRpcFactory; +import com.google.cloud.storage.StorageOptions; + +/** + * Utility to create an in-memory storage configuration for testing. Storage options can be obtained + * via the {@link #getOptions()} method. Returned options will point to FakeStorageRpc. + * + *

Note, the created in-memory storage configuration supports limited set of operations and is + * not thread-safe: + * + *

    + *
  • Supported operations + *
      + *
    • object create + *
    • object get + *
    • object delete + *
    • list the contents of a bucket + *
    + *
  • Unsupported operations + *
      + *
    • bucket create + *
    • bucket get + *
    • bucket delete + *
    • list all buckets + *
    • generations + *
    • file attributes + *
    • patch + *
    • continueRewrite + *
    • createBatch + *
    • checksums, etags + *
    • IAM operations + *
    + *
+ * + * {@link FakeStorageRpc#list(String, java.util.Map)} lists all the objects that have been created + * rather than the objects in the provided bucket. Since this class does not support creating, + * listing and deleting buckets, the parameter bucket here is not actually used and on serves as a + * placeholder. + */ +public final class LocalStorageHelper { + + // used for testing. Will throw if you pass it an option. + private static final FakeStorageRpc instance = new FakeStorageRpc(true); + + private LocalStorageHelper() {} + + /** + * Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, and resets it + * first so you start from a clean slate. That instance will throw if you pass it any option. + */ + public static StorageOptions getOptions() { + instance.reset(); + return StorageOptions.newBuilder() + .setProjectId("fake-project-for-testing") + .setServiceRpcFactory(new FakeStorageRpcFactory()) + .build(); + } + + /** + * Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance with the given + * option. + */ + public static StorageOptions customOptions(final boolean throwIfOptions) { + return StorageOptions.newBuilder() + .setProjectId("fake-project-for-testing") + .setServiceRpcFactory(new FakeStorageRpcFactory(new FakeStorageRpc(throwIfOptions))) + .build(); + } + + public static class FakeStorageRpcFactory implements ServiceRpcFactory { + + private final FakeStorageRpc instance; + + public FakeStorageRpcFactory() { + this(LocalStorageHelper.instance); + } + + public FakeStorageRpcFactory(FakeStorageRpc instance) { + this.instance = instance; + } + + @Override + public ServiceRpc create(StorageOptions storageOptions) { + return instance; + } + } +} diff --git a/core/src/test/java/google/registry/gcs/backport/README.md b/core/src/test/java/google/registry/gcs/backport/README.md new file mode 100644 index 000000000..caa714b7e --- /dev/null +++ b/core/src/test/java/google/registry/gcs/backport/README.md @@ -0,0 +1,9 @@ +The files here are directly lifted +from [googleapis/java-storage-nio](https://github.com/googleapis/java-storage-nio/tree/master/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing) +. They are needed because the `StorageOptions` returned from `LocalStorageHelper.getOptions()` +should be serializable for the `GcsUtils` class itself to be serializable in tests, but is not. The +bug is [fixed](https://github.com/googleapis/java-storage-nio/pull/606) upstream. However, the +current released package does not contain the fix yet. + +They are not put under `common/testing` because we do not want to introduce a dependency on the +testing configuration from core. \ No newline at end of file diff --git a/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java b/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java index 229b49827..426d90d81 100644 --- a/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java +++ b/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java @@ -16,23 +16,20 @@ package google.registry.rde; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.SystemInfo.hasCommand; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assumptions.assumeTrue; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.io.ByteSource; import com.google.common.io.CharStreams; import com.google.common.io.Files; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.Keyring; import google.registry.testing.AppEngineExtension; import google.registry.testing.BouncyCastleProviderExtension; import google.registry.testing.FakeKeyringModule; -import google.registry.testing.GcsTestingUtils; import google.registry.testing.GpgSystemCommandExtension; import java.io.File; import java.io.IOException; @@ -52,14 +49,12 @@ public class BrdaCopyActionTest { private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml"); - private static final GcsFilename STAGE_FILE = - new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde"); - private static final GcsFilename STAGE_LENGTH_FILE = - new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.length"); - private static final GcsFilename RYDE_FILE = - new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.ryde"); - private static final GcsFilename SIG_FILE = - new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.sig"); + private static final BlobId STAGE_FILE = + BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde"); + private static final BlobId STAGE_LENGTH_FILE = + BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.length"); + private static final BlobId RYDE_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.ryde"); + private static final BlobId SIG_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.sig"); @RegisterExtension public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension(); @@ -89,8 +84,7 @@ public class BrdaCopyActionTest { } } - private final GcsService gcsService = GcsServiceFactory.createGcsService(); - private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final BrdaCopyAction action = new BrdaCopyAction(); @BeforeEach @@ -105,9 +99,8 @@ public class BrdaCopyActionTest { action.stagingDecryptionKey = decryptKey; byte[] xml = DEPOSIT_XML.read(); - GcsTestingUtils.writeGcsFile(gcsService, STAGE_FILE, Ghostryde.encode(xml, encryptKey)); - GcsTestingUtils.writeGcsFile(gcsService, STAGE_LENGTH_FILE, - Long.toString(xml.length).getBytes(UTF_8)); + gcsUtils.createFromBytes(STAGE_FILE, Ghostryde.encode(xml, encryptKey)); + gcsUtils.createFromBytes(STAGE_LENGTH_FILE, Long.toString(xml.length).getBytes(UTF_8)); } @Test @@ -124,7 +117,7 @@ public class BrdaCopyActionTest { action.run(); File rydeTmp = new File(gpg.getCwd(), "ryde"); - Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp); + Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp); Process pid = gpg.exec( "gpg", @@ -172,8 +165,8 @@ public class BrdaCopyActionTest { File rydeTmp = new File(gpg.getCwd(), "ryde"); File sigTmp = new File(gpg.getCwd(), "ryde.sig"); - Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp); - Files.write(readGcsFile(gcsService, SIG_FILE), sigTmp); + Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp); + Files.write(gcsUtils.readBytesFrom(SIG_FILE), sigTmp); Process pid = gpg.exec("gpg", "--verify", sigTmp.toString(), rydeTmp.toString()); String stderr = slurp(pid.getErrorStream()); diff --git a/core/src/test/java/google/registry/rde/RdeReportActionTest.java b/core/src/test/java/google/registry/rde/RdeReportActionTest.java index 81f06d72c..eda53a3de 100644 --- a/core/src/test/java/google/registry/rde/RdeReportActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeReportActionTest.java @@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.loadByKey; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.deleteGcsFile; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.joda.time.Duration.standardDays; @@ -41,13 +39,12 @@ import com.google.appengine.api.urlfetch.HTTPHeader; import com.google.appengine.api.urlfetch.HTTPRequest; import com.google.appengine.api.urlfetch.HTTPResponse; import com.google.appengine.api.urlfetch.URLFetchService; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.common.Cursor; import google.registry.model.rde.RdeRevision; import google.registry.model.registry.Registry; @@ -95,9 +92,9 @@ public class RdeReportActionTest { private final ArgumentCaptor request = ArgumentCaptor.forClass(HTTPRequest.class); private final HTTPResponse httpResponse = mock(HTTPResponse.class); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); - private final GcsFilename reportFile = - new GcsFilename("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde"); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); + private final BlobId reportFile = + BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde"); private RdeReportAction createAction() { RdeReporter reporter = new RdeReporter(); @@ -106,7 +103,7 @@ public class RdeReportActionTest { reporter.password = "foo"; reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); RdeReportAction action = new RdeReportAction(); - action.gcsUtils = new GcsUtils(gcsService, 1024); + action.gcsUtils = gcsUtils; action.response = response; action.bucket = "tub"; action.tld = "test"; @@ -126,7 +123,7 @@ public class RdeReportActionTest { Cursor.create(RDE_REPORT, DateTime.parse("2006-06-06TZ"), Registry.get("test"))); persistResource( Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test"))); - writeGcsFile(gcsService, reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0)); } @@ -169,11 +166,10 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_regeneratedReport() throws Exception { - deleteGcsFile(gcsService, reportFile); - GcsFilename newReport = - new GcsFilename("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde"); + gcsUtils.delete(reportFile); + BlobId newReport = BlobId.of("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde"); PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); - writeGcsFile(gcsService, newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1)); when(httpResponse.getResponseCode()).thenReturn(SC_OK); when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); diff --git a/core/src/test/java/google/registry/rde/RdeStagingActionTest.java b/core/src/test/java/google/registry/rde/RdeStagingActionTest.java index 01d46d084..1b9a72a87 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingActionTest.java @@ -14,7 +14,6 @@ package google.registry.rde; -import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.truth.Truth.assertThat; import static google.registry.model.common.Cursor.CursorType.BRDA; import static google.registry.model.common.Cursor.CursorType.RDE_STAGING; @@ -26,7 +25,6 @@ import static google.registry.rde.RdeFixtures.makeHostResource; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DatabaseHelper.persistResourceWithCommitLog; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.TaskQueueHelper.assertAtLeastOneTaskIsEnqueued; import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; @@ -36,16 +34,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Arrays.asList; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; -import com.google.appengine.tools.cloudstorage.ListItem; -import com.google.appengine.tools.cloudstorage.ListOptions; -import com.google.appengine.tools.cloudstorage.ListResult; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.Keyring; import google.registry.keyring.api.PgpHelper; import google.registry.model.common.Cursor; @@ -98,16 +93,16 @@ import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link RdeStagingAction}. */ public class RdeStagingActionTest extends MapreduceTestCase { - private static final GcsFilename XML_FILE = - new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde"); - private static final GcsFilename LENGTH_FILE = - new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length"); + private static final BlobId XML_FILE = + BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde"); + private static final BlobId LENGTH_FILE = + BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length"); @RegisterExtension public final InjectExtension inject = new InjectExtension(); private final FakeClock clock = new FakeClock(); private final FakeResponse response = new FakeResponse(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final List alreadyExtracted = new ArrayList<>(); private static PGPPublicKey encryptKey; @@ -131,7 +126,6 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.reducerFactory = new RdeStagingReducer.Factory(); action.reducerFactory.taskQueueUtils = new TaskQueueUtils(new Retrier(new SystemSleeper(), 1)); action.reducerFactory.lockHandler = new FakeLockHandler(true); - action.reducerFactory.gcsBufferSize = 0; action.reducerFactory.bucket = "rde-bucket"; action.reducerFactory.lockTimeout = Duration.standardHours(1); action.reducerFactory.stagingKeyBytes = PgpHelper.convertPublicKeyToBytes(encryptKey); @@ -140,6 +134,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.pendingDepositChecker.brdaInterval = Duration.standardDays(7); action.pendingDepositChecker.clock = clock; action.pendingDepositChecker.rdeInterval = Duration.standardDays(1); + action.gcsUtils = gcsUtils; action.response = response; action.transactionCooldown = Duration.ZERO; action.directory = Optional.empty(); @@ -327,7 +322,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey)); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); assertThat(header.getTld()).isEqualTo("lol"); @@ -358,7 +353,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey)); assertThat(deposit.getType()).isEqualTo(XjcRdeDepositTypeType.FULL); assertThat(deposit.getId()).isEqualTo(RdeUtil.timestampToId(DateTime.parse("2000-01-01TZ"))); assertThat(deposit.getWatermark()).isEqualTo(DateTime.parse("2000-01-01TZ")); @@ -400,7 +395,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey)); XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); @@ -494,12 +489,13 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - for (GcsFilename filename : asList( - new GcsFilename("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"), - new GcsFilename("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) { + for (BlobId filename : + asList( + BlobId.of("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"), + BlobId.of("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey)); XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); @@ -536,10 +532,10 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - GcsFilename filename = new GcsFilename("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde"); + BlobId filename = BlobId.of("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde"); XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey)); XjcRdeDomain domain = extractAndRemoveContentWithType(XjcRdeDomain.class, deposit); XjcRdeIdn firstIdn = extractAndRemoveContentWithType(XjcRdeIdn.class, deposit); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); @@ -579,8 +575,8 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - byte[] deposit = Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey); - assertThat(Integer.parseInt(new String(readGcsFile(gcsService, LENGTH_FILE), UTF_8))) + byte[] deposit = Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey); + assertThat(Integer.parseInt(new String(gcsUtils.readBytesFrom(LENGTH_FILE), UTF_8))) .isEqualTo(deposit.length); } @@ -781,23 +777,20 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - ListResult listResult = - gcsService.list("rde-bucket", new ListOptions.Builder().setPrefix("manual/test").build()); - ImmutableSet filenames = - ImmutableList.copyOf(listResult).stream().map(ListItem::getName).collect(toImmutableSet()); + ImmutableList filenames = gcsUtils.listFolderObjects("rde-bucket", "manual/test/"); for (String tld : tlds) { assertThat(filenames) .containsAtLeast( - "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde", - "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length", - "manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length", - "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde", - "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length", - "manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length"); + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde", + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length", + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length", + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde", + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length", + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length"); assertThat( auditedOfy() @@ -837,8 +830,8 @@ public class RdeStagingActionTest extends MapreduceTestCase { } private String readXml(String objectName) throws IOException, PGPException { - GcsFilename file = new GcsFilename("rde-bucket", objectName); - return new String(Ghostryde.decode(readGcsFile(gcsService, file), decryptKey), UTF_8); + BlobId file = BlobId.of("rde-bucket", objectName); + return new String(Ghostryde.decode(gcsUtils.readBytesFrom(file), decryptKey), UTF_8); } private diff --git a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java index 3014091f9..f2fa21bef 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java @@ -20,18 +20,18 @@ import static google.registry.model.rde.RdeMode.FULL; import static google.registry.model.rde.RdeMode.THIN; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; import static google.registry.util.ResourceUtils.readResourceUtf8; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.StorageException; import com.google.common.collect.ImmutableList; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.PgpHelper; import google.registry.model.common.Cursor; import google.registry.model.common.Cursor.CursorType; @@ -67,7 +67,7 @@ class RdeStagingReducerTest { AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); private static final String GCS_BUCKET = "test-rde-bucket"; - private static final GcsService gcsService = GcsServiceFactory.createGcsService(); + private static final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private static final PGPPrivateKey decryptionKey = new FakeKeyringModule().get().getRdeStagingDecryptionKey(); private static final PGPPublicKey encryptionKey = @@ -95,11 +95,11 @@ class RdeStagingReducerTest { new RdeStagingReducer( new TaskQueueUtils(new Retrier(new FakeSleeper(new FakeClock()), 1)), new FakeLockHandler(true), - 1024, GCS_BUCKET, Duration.ZERO, PgpHelper.convertPublicKeyToBytes(encryptionKey), - ValidationMode.STRICT); + ValidationMode.STRICT, + gcsUtils); @BeforeEach void beforeEach() { @@ -125,11 +125,10 @@ class RdeStagingReducerTest { compareLength(outputFile, "soy_2000-01-01_thin_S1_R1.xml.length"); // BRDA doesn't write a report file. assertThrows( - IOException.class, + StorageException.class, () -> - readGcsFile( - gcsService, - new GcsFilename(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde"))); + gcsUtils.readBytesFrom( + BlobId.of(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde"))); assertThat(loadCursorTime(CursorType.BRDA)) .isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1))); assertThat(loadRevision(THIN)).isEqualTo(1); @@ -153,12 +152,10 @@ class RdeStagingReducerTest { compareLength(outputFile, "manual/soy_2000-01-01_thin_S1_R0.xml.length"); // BRDA doesn't write a report file. assertThrows( - IOException.class, + StorageException.class, () -> - readGcsFile( - gcsService, - new GcsFilename( - GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde"))); + gcsUtils.readBytesFrom( + BlobId.of(GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde"))); // No extra operations in manual mode. assertThat(loadCursorTime(CursorType.BRDA)).isEquivalentAccordingToCompareTo(now); assertThat(loadRevision(THIN)).isEqualTo(0); @@ -210,8 +207,7 @@ class RdeStagingReducerTest { private static void compareLength(String outputFile, String lengthFilename) throws IOException { assertThat(String.valueOf(outputFile.getBytes(UTF_8).length)) .isEqualTo( - new String( - readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, lengthFilename)), UTF_8)); + new String(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, lengthFilename)), UTF_8)); } private static DateTime loadCursorTime(CursorType type) { @@ -233,8 +229,7 @@ class RdeStagingReducerTest { private static String decryptGhostrydeGcsFile(String filename) throws IOException, PGPException { return new String( - Ghostryde.decode( - readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, filename)), decryptionKey), + Ghostryde.decode(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, filename)), decryptionKey), UTF_8); } diff --git a/core/src/test/java/google/registry/rde/RdeUploadActionTest.java b/core/src/test/java/google/registry/rde/RdeUploadActionTest.java index 1281ed920..271f669d6 100644 --- a/core/src/test/java/google/registry/rde/RdeUploadActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeUploadActionTest.java @@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DatabaseHelper.persistSimpleResource; -import static google.registry.testing.GcsTestingUtils.readGcsFile; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static google.registry.testing.SystemInfo.hasCommand; import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; @@ -45,9 +43,7 @@ import static org.mockito.Mockito.when; import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.utils.SystemProperty; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.io.ByteSource; import com.google.common.io.CharStreams; import com.google.common.io.Files; @@ -55,6 +51,7 @@ import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.Keyring; import google.registry.model.common.Cursor; import google.registry.model.common.Cursor.CursorType; @@ -94,23 +91,24 @@ import org.mockito.stubbing.OngoingStubbing; @DualDatabaseTest public class RdeUploadActionTest { - private static final int BUFFER_SIZE = 64 * 1024; private static final ByteSource REPORT_XML = RdeTestData.loadBytes("report.xml"); private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml"); - private static final GcsFilename GHOSTRYDE_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde"); - private static final GcsFilename LENGTH_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.length"); - private static final GcsFilename REPORT_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde"); + private static final BlobId GHOSTRYDE_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde"); + private static final BlobId LENGTH_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.length"); + private static final BlobId REPORT_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde"); - private static final GcsFilename GHOSTRYDE_R1_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde"); - private static final GcsFilename LENGTH_R1_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.length"); - private static final GcsFilename REPORT_R1_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde"); + private static final BlobId GHOSTRYDE_R1_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde"); + private static final BlobId LENGTH_R1_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.length"); + private static final BlobId REPORT_R1_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde"); + + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @RegisterExtension final SftpServerExtension sftpd = new SftpServerExtension(); @@ -139,7 +137,7 @@ public class RdeUploadActionTest { try (Keyring keyring = new FakeKeyringModule().get()) { RdeUploadAction action = new RdeUploadAction(); action.clock = clock; - action.gcsUtils = new GcsUtils(gcsService, BUFFER_SIZE); + action.gcsUtils = gcsUtils; action.lazyJsch = () -> JSchModule.provideJSch( @@ -176,22 +174,20 @@ public class RdeUploadActionTest { return jschSpy; } - private GcsService gcsService; @BeforeEach void beforeEach() throws Exception { // Force "development" mode so we don't try to really connect to GCS. SystemProperty.environment.set(SystemProperty.Environment.Value.Development); - gcsService = GcsServiceFactory.createGcsService(); createTld("tld"); PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); - writeGcsFile(gcsService, GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); - writeGcsFile(gcsService, GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); - writeGcsFile(gcsService, LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); - writeGcsFile(gcsService, LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); - writeGcsFile(gcsService, REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); - writeGcsFile(gcsService, REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); + gcsUtils.createFromBytes(LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); + gcsUtils.createFromBytes(REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm() .transact( () -> { @@ -271,9 +267,9 @@ public class RdeUploadActionTest { String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde"; String sigFilename = "tld_2010-10-17_full_S1_R0.sig"; assertThat(folder.list()).asList().containsExactly(rydeFilename, sigFilename); - assertThat(readGcsFile(gcsService, new GcsFilename("bucket", rydeFilename))) + assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", rydeFilename))) .isEqualTo(Files.toByteArray(new File(folder, rydeFilename))); - assertThat(readGcsFile(gcsService, new GcsFilename("bucket", sigFilename))) + assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", sigFilename))) .isEqualTo(Files.toByteArray(new File(folder, sigFilename))); } diff --git a/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java b/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java index b4089250c..e43dbd22f 100644 --- a/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java +++ b/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java @@ -23,7 +23,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.net.MediaType; import google.registry.gcs.GcsUtils; @@ -51,7 +51,7 @@ class BillingEmailUtilsTest { void beforeEach() throws Exception { emailService = mock(SendEmailService.class); gcsUtils = mock(GcsUtils.class); - when(gcsUtils.openInputStream(new GcsFilename("test-bucket", "results/REG-INV-2017-10.csv"))) + when(gcsUtils.openInputStream(BlobId.of("test-bucket", "results/REG-INV-2017-10.csv"))) .thenReturn( new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8))); contentCaptor = ArgumentCaptor.forClass(EmailMessage.class); diff --git a/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java b/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java index 4b63f8170..fc2d79f7a 100644 --- a/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java +++ b/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java @@ -17,7 +17,6 @@ package google.registry.reporting.billing; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatabaseHelper.loadRegistrar; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.mockito.ArgumentMatchers.any; @@ -29,11 +28,10 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.net.MediaType; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.storage.drive.DriveConnection; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; @@ -52,8 +50,7 @@ class CopyDetailReportsActionTest { final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); - private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private FakeResponse response; private DriveConnection driveConnection; @@ -80,14 +77,12 @@ class CopyDetailReportsActionTest { @Test void testSuccess() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"), "hello,world\n1,2".getBytes(UTF_8)); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); action.run(); @@ -111,14 +106,12 @@ class CopyDetailReportsActionTest { @Test void testSuccess_nonDetailReportFiles_notSent() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"), "hello,world\n1,2".getBytes(UTF_8)); action.run(); verify(driveConnection) @@ -136,9 +129,8 @@ class CopyDetailReportsActionTest { @Test void testSuccess_transientIOException_retries() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); when(driveConnection.createOrUpdateFile(any(), any(), any(), any())) .thenThrow(new IOException("expected")) @@ -158,13 +150,12 @@ class CopyDetailReportsActionTest { @Test void testFail_tooManyFailures_sendsAlertEmail_continues() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"), + + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"), "hello,world\n1,2".getBytes(UTF_8)); when(driveConnection.createOrUpdateFile( eq("invoice_details_2017-10_TheRegistrar_hello.csv"), any(), any(), any())) @@ -201,9 +192,8 @@ class CopyDetailReportsActionTest { @Test void testFail_registrarDoesntExist_doesntCopy() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); action.run(); verifyNoInteractions(driveConnection); @@ -212,9 +202,8 @@ class CopyDetailReportsActionTest { @Test void testFail_noRegistrarFolderId_doesntCopy() throws IOException { persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId(null).build()); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); action.run(); verifyNoInteractions(driveConnection); diff --git a/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java b/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java index edf2ede40..716ece531 100644 --- a/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java +++ b/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java @@ -15,16 +15,13 @@ package google.registry.reporting.icann; import static com.google.common.truth.Truth.assertThat; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableTable; import com.google.common.util.concurrent.ListenableFuture; @@ -32,6 +29,7 @@ import google.registry.bigquery.BigqueryConnection; import google.registry.bigquery.BigqueryConnection.DestinationTable; import google.registry.bigquery.BigqueryUtils.TableType; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.reporting.icann.IcannReportingModule.ReportType; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeResponse; @@ -46,9 +44,9 @@ class IcannReportingStagerTest { private BigqueryConnection bigquery = mock(BigqueryConnection.class); FakeResponse response = new FakeResponse(); - private GcsService gcsService = GcsServiceFactory.createGcsService(); private YearMonth yearMonth = new YearMonth(2017, 6); private String subdir = "icann/monthly/2017-06"; + private GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @RegisterExtension final AppEngineExtension appEngine = @@ -65,7 +63,7 @@ class IcannReportingStagerTest { action.transactionsQueryBuilder = transactionsBuilder; action.reportingBucket = "test-bucket"; action.bigquery = bigquery; - action.gcsUtils = new GcsUtils(gcsService, 1024); + action.gcsUtils = gcsUtils; return action; } @@ -100,14 +98,12 @@ class IcannReportingStagerTest { String expectedReport1 = "fooField,barField\r\n12,34"; String expectedReport2 = "fooField,barField\r\n56,78"; byte[] generatedFile1 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv")); assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1); byte[] generatedFile2 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv")); assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2); } @@ -144,14 +140,12 @@ class IcannReportingStagerTest { "registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30"; String expectedReport2 = "registrar,iana,field\r\n\"reg1\",123,30\r\nTotals,,30"; byte[] generatedFile1 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv")); assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1); byte[] generatedFile2 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv")); assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2); } @@ -164,8 +158,7 @@ class IcannReportingStagerTest { String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n"; byte[] generatedManifest = - readGcsFile( - gcsService, new GcsFilename("test-bucket/icann/monthly/2017-06", "MANIFEST.txt")); + gcsUtils.readBytesFrom(BlobId.of("test-bucket/icann/monthly/2017-06", "MANIFEST.txt")); assertThat(new String(generatedManifest, UTF_8)).isEqualTo(expectedManifest); } diff --git a/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java b/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java index 3c4203975..68be3ed91 100644 --- a/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java +++ b/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java @@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.createTlds; import static google.registry.testing.DatabaseHelper.loadByKey; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static google.registry.testing.LogsSubject.assertAboutLogs; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -29,11 +28,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.testing.TestLogHandler; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.common.Cursor; import google.registry.model.common.Cursor.CursorType; import google.registry.model.registry.Registry; @@ -69,7 +67,7 @@ class IcannReportingUploadActionTest { private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class); private final SendEmailService emailService = mock(SendEmailService.class); private final FakeResponse response = new FakeResponse(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final TestLogHandler logHandler = new TestLogHandler(); private final Logger loggerToIntercept = Logger.getLogger(IcannReportingUploadAction.class.getCanonicalName()); @@ -78,7 +76,7 @@ class IcannReportingUploadActionTest { private IcannReportingUploadAction createAction() throws Exception { IcannReportingUploadAction action = new IcannReportingUploadAction(); action.icannReporter = mockReporter; - action.gcsUtils = new GcsUtils(gcsService, 1024); + action.gcsUtils = gcsUtils; action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); action.reportingBucket = "basin"; action.emailService = emailService; @@ -93,22 +91,14 @@ class IcannReportingUploadActionTest { @BeforeEach void beforeEach() throws Exception { createTlds("tld", "foo"); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"), - PAYLOAD_SUCCESS); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), - PAYLOAD_FAIL); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"), - PAYLOAD_SUCCESS); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "foo-activity-200606.csv"), - PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"), PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_FAIL); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"), PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "foo-activity-200606.csv"), PAYLOAD_SUCCESS); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv")).thenReturn(true); when(mockReporter.send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv")).thenReturn(true); when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv")).thenReturn(false); @@ -161,14 +151,10 @@ class IcannReportingUploadActionTest { persistResource( Cursor.create( CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-01-01TZ"), Registry.get("tld"))); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"), - PAYLOAD_SUCCESS); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2005-12", "tld-activity-200512.csv"), - PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"), PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2005-12", "tld-activity-200512.csv"), PAYLOAD_SUCCESS); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200512.csv")).thenReturn(true); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200512.csv")).thenReturn(true); @@ -191,10 +177,8 @@ class IcannReportingUploadActionTest { @TestOfyAndSql void testSuccess_advancesCursor() throws Exception { - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), - PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_SUCCESS); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true); IcannReportingUploadAction action = createAction(); action.run(); diff --git a/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java b/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java index 4d7136646..c0702d0e7 100644 --- a/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java +++ b/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java @@ -19,7 +19,7 @@ import static com.google.common.truth.Truth8.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -133,8 +133,8 @@ public class Spec11RegistrarThreatMatchesParserTest { } private void setupFile(String fileWithContent, String fileDate) { - GcsFilename gcsFilename = - new GcsFilename( + BlobId gcsFilename = + BlobId.of( "test-bucket", String.format("icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_%s", fileDate)); when(gcsUtils.existsAndNotEmpty(gcsFilename)).thenReturn(true); diff --git a/core/src/test/java/google/registry/testing/GcsTestingUtils.java b/core/src/test/java/google/registry/testing/GcsTestingUtils.java deleted file mode 100644 index a51bb3dfe..000000000 --- a/core/src/test/java/google/registry/testing/GcsTestingUtils.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.testing; - -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.common.io.ByteStreams; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; - -/** Utility methods for testing Google Cloud Storage code. */ -public final class GcsTestingUtils { - - /** Slurps a Cloud Storage file into memory. */ - public static byte[] readGcsFile(GcsService gcsService, GcsFilename file) - throws IOException { - try (InputStream input = Channels.newInputStream(gcsService.openReadChannel(file, 0))) { - return ByteStreams.toByteArray(input); - } - } - - /** Writes a Cloud Storage file. */ - public static void writeGcsFile(GcsService gcsService, GcsFilename file, byte[] data) - throws IOException { - gcsService.createOrReplace(file, GcsFileOptions.getDefaultInstance(), ByteBuffer.wrap(data)); - } - - public static void deleteGcsFile(GcsService gcsService, GcsFilename file) throws IOException { - gcsService.delete(file); - } - - private GcsTestingUtils() {} -} diff --git a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java index c65a02864..d771d5e0e 100644 --- a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java @@ -14,7 +14,6 @@ package google.registry.tools.server; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatabaseHelper.createTlds; import static google.registry.testing.DatabaseHelper.newDomainBase; @@ -23,17 +22,17 @@ import static google.registry.testing.DatabaseHelper.persistActiveContact; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistActiveHost; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.TestDataHelper.loadFile; import static java.nio.charset.StandardCharsets.UTF_8; import static org.joda.time.Duration.standardDays; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobId; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.domain.secdns.DelegationSignerData; import google.registry.model.eppcommon.StatusValue; import google.registry.model.host.HostResource; @@ -50,7 +49,7 @@ import org.junit.jupiter.api.Test; /** Tests for {@link GenerateZoneFilesAction}. */ class GenerateZoneFilesActionTest extends MapreduceTestCase { - private final GcsService gcsService = createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @Test void testGenerate() throws Exception { @@ -114,7 +113,7 @@ class GenerateZoneFilesActionTest extends MapreduceTestCase