From 676616a1721363bfc2ad98078b789ef0cda04d0c Mon Sep 17 00:00:00 2001 From: Lai Jiang Date: Tue, 13 Jul 2021 14:52:37 -0400 Subject: [PATCH] Remove the use of GCS APIs provided from GAE SDK (#1228) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The API provided by the GAE SDK will not be available outside GAE runtime. This presents a problem when we migrate off of GAE. More pressingly, the RDE pipeline migration to Beam requires that we write to GCS on GCE. Previously we were able to sidestep the issue by delegating the writes to FileIO provided by Beam, which knows how to write to GCS. However the RDE pipeline cannot use FileIO directly as it needs to write to multiple files in one go and explicit use of GCS API is needed. An unfortunate side effect of the API migration is that the new testing library contains a bug which makes serializing GcsUtils impossible. It is fixed upstream but not released yet. The fix has been backported for the time being. This change is [Reviewable](https://reviewable.io/reviews/google/nomulus/1228) --- .../gradle/dependency-locks/compile.lockfile | 4 +- .../compileClasspath.lockfile | 4 +- .../dependency-locks/deploy_jar.lockfile | 4 +- .../gradle/dependency-locks/runtime.lockfile | 4 +- .../runtimeClasspath.lockfile | 4 +- .../dependency-locks/testCompile.lockfile | 2 +- .../testCompileClasspath.lockfile | 2 +- .../dependency-locks/testRuntime.lockfile | 2 +- .../testRuntimeClasspath.lockfile | 2 +- .../gradle/dependency-locks/compile.lockfile | 6 +- .../compileClasspath.lockfile | 6 +- .../gradle/dependency-locks/default.lockfile | 6 +- .../dependency-locks/deploy_jar.lockfile | 6 +- .../gradle/dependency-locks/runtime.lockfile | 6 +- .../runtimeClasspath.lockfile | 6 +- .../dependency-locks/testCompile.lockfile | 2 +- .../testCompileClasspath.lockfile | 2 +- .../dependency-locks/testRuntime.lockfile | 2 +- .../testRuntimeClasspath.lockfile | 2 +- .../dependency-locks/testingCompile.lockfile | 2 +- .../testingCompileClasspath.lockfile | 2 +- .../dependency-locks/testingRuntime.lockfile | 2 +- .../testingRuntimeClasspath.lockfile | 2 +- config/presubmits.py | 4 +- core/build.gradle | 4 + .../annotationProcessor.lockfile | 6 +- core/gradle/dependency-locks/compile.lockfile | 19 +- .../compileClasspath.lockfile | 19 +- core/gradle/dependency-locks/default.lockfile | 19 +- .../dependency-locks/deploy_jar.lockfile | 19 +- .../dependency-locks/nonprodCompile.lockfile | 19 +- .../nonprodCompileClasspath.lockfile | 19 +- .../dependency-locks/nonprodRuntime.lockfile | 19 +- .../nonprodRuntimeClasspath.lockfile | 19 +- core/gradle/dependency-locks/runtime.lockfile | 19 +- .../runtimeClasspath.lockfile | 19 +- .../testAnnotationProcessor.lockfile | 6 +- .../dependency-locks/testCompile.lockfile | 62 ++- .../testCompileClasspath.lockfile | 62 ++- .../dependency-locks/testRuntime.lockfile | 62 ++- .../testRuntimeClasspath.lockfile | 62 ++- .../backup/ExportCommitLogDiffAction.java | 24 +- .../registry/backup/GcsDiffFileLister.java | 75 ++- .../backup/ReplayCommitLogsToSqlAction.java | 41 +- .../backup/RestoreCommitLogsAction.java | 19 +- .../registry/config/RegistryConfig.java | 11 - .../export/ExportDomainListsAction.java | 26 +- .../google/registry/gcs/GcsServiceModule.java | 34 -- .../java/google/registry/gcs/GcsUtils.java | 148 +++-- .../module/backend/BackendComponent.java | 2 - .../registry/module/tools/ToolsComponent.java | 2 - .../google/registry/rde/BrdaCopyAction.java | 12 +- .../google/registry/rde/RdeReportAction.java | 6 +- .../google/registry/rde/RdeStagingAction.java | 4 +- .../registry/rde/RdeStagingReducer.java | 33 +- .../google/registry/rde/RdeUploadAction.java | 49 +- .../reporting/billing/BillingEmailUtils.java | 5 +- .../billing/CopyDetailReportsAction.java | 4 +- .../reporting/icann/IcannReportingStager.java | 6 +- .../icann/IcannReportingUploadAction.java | 12 +- .../Spec11RegistrarThreatMatchesParser.java | 10 +- .../tools/server/GenerateZoneFilesAction.java | 22 +- .../backup/ExportCommitLogDiffActionTest.java | 83 ++- .../backup/GcsDiffFileListerTest.java | 91 +-- .../ReplayCommitLogsToSqlActionTest.java | 52 +- .../backup/RestoreCommitLogsActionTest.java | 71 ++- .../export/ExportDomainListsActionTest.java | 47 +- .../google/registry/gcs/GcsUtilsTest.java | 128 +++++ .../registry/gcs/backport/FakeStorageRpc.java | 517 ++++++++++++++++++ .../gcs/backport/LocalStorageHelper.java | 106 ++++ .../google/registry/gcs/backport/README.md | 9 + .../registry/rde/BrdaCopyActionTest.java | 35 +- .../registry/rde/RdeReportActionTest.java | 24 +- .../registry/rde/RdeStagingActionTest.java | 75 ++- .../registry/rde/RdeStagingReducerTest.java | 35 +- .../registry/rde/RdeUploadActionTest.java | 54 +- .../billing/BillingEmailUtilsTest.java | 4 +- .../billing/CopyDetailReportsActionTest.java | 55 +- .../icann/IcannReportingStagerTest.java | 33 +- .../icann/IcannReportingUploadActionTest.java | 52 +- ...pec11RegistrarThreatMatchesParserTest.java | 6 +- .../registry/testing/GcsTestingUtils.java | 48 -- .../server/GenerateZoneFilesActionTest.java | 16 +- .../dependency-locks/testCompile.lockfile | 2 +- .../testCompileClasspath.lockfile | 2 +- .../dependency-locks/testRuntime.lockfile | 2 +- .../testRuntimeClasspath.lockfile | 2 +- dependencies.gradle | 8 +- docs/gradle/dependency-locks/compile.lockfile | 19 +- .../compileClasspath.lockfile | 19 +- docs/gradle/dependency-locks/default.lockfile | 19 +- .../dependency-locks/deploy_jar.lockfile | 19 +- docs/gradle/dependency-locks/runtime.lockfile | 19 +- .../runtimeClasspath.lockfile | 19 +- .../dependency-locks/testCompile.lockfile | 17 +- .../testCompileClasspath.lockfile | 17 +- .../dependency-locks/testRuntime.lockfile | 17 +- .../testRuntimeClasspath.lockfile | 17 +- .../gradle/dependency-locks/compile.lockfile | 6 +- .../compileClasspath.lockfile | 6 +- .../gradle/dependency-locks/default.lockfile | 6 +- .../dependency-locks/deploy_jar.lockfile | 6 +- .../gradle/dependency-locks/runtime.lockfile | 6 +- .../runtimeClasspath.lockfile | 6 +- .../dependency-locks/testCompile.lockfile | 2 +- .../testCompileClasspath.lockfile | 2 +- .../dependency-locks/testRuntime.lockfile | 2 +- .../testRuntimeClasspath.lockfile | 2 +- .../gradle/dependency-locks/compile.lockfile | 6 +- .../compileClasspath.lockfile | 6 +- .../gradle/dependency-locks/default.lockfile | 6 +- .../dependency-locks/deploy_jar.lockfile | 6 +- .../gradle/dependency-locks/runtime.lockfile | 6 +- .../runtimeClasspath.lockfile | 6 +- .../dependency-locks/testCompile.lockfile | 2 +- .../testCompileClasspath.lockfile | 2 +- .../dependency-locks/testRuntime.lockfile | 2 +- .../testRuntimeClasspath.lockfile | 2 +- .../gradle/dependency-locks/compile.lockfile | 6 +- .../compileClasspath.lockfile | 6 +- .../gradle/dependency-locks/default.lockfile | 6 +- .../dependency-locks/deploy_jar.lockfile | 6 +- .../gradle/dependency-locks/runtime.lockfile | 6 +- .../runtimeClasspath.lockfile | 6 +- .../dependency-locks/testCompile.lockfile | 6 +- .../testCompileClasspath.lockfile | 6 +- .../dependency-locks/testRuntime.lockfile | 6 +- .../testRuntimeClasspath.lockfile | 6 +- proxy/build.gradle | 4 +- .../gradle/dependency-locks/compile.lockfile | 28 +- .../compileClasspath.lockfile | 28 +- .../gradle/dependency-locks/default.lockfile | 28 +- .../dependency-locks/deploy_jar.lockfile | 28 +- .../gradle/dependency-locks/runtime.lockfile | 28 +- .../runtimeClasspath.lockfile | 28 +- .../dependency-locks/testCompile.lockfile | 24 +- .../testCompileClasspath.lockfile | 24 +- .../dependency-locks/testRuntime.lockfile | 24 +- .../testRuntimeClasspath.lockfile | 24 +- .../google/registry/proxy/ProxyModule.java | 33 +- .../gradle/dependency-locks/compile.lockfile | 19 +- .../compileClasspath.lockfile | 19 +- .../gradle/dependency-locks/default.lockfile | 19 +- .../gradle/dependency-locks/runtime.lockfile | 19 +- .../runtimeClasspath.lockfile | 19 +- .../dependency-locks/testCompile.lockfile | 19 +- .../testCompileClasspath.lockfile | 19 +- .../dependency-locks/testRuntime.lockfile | 19 +- .../testRuntimeClasspath.lockfile | 19 +- .../gradle/dependency-locks/compile.lockfile | 19 +- .../compileClasspath.lockfile | 19 +- .../gradle/dependency-locks/default.lockfile | 19 +- .../gradle/dependency-locks/runtime.lockfile | 19 +- .../runtimeClasspath.lockfile | 19 +- .../dependency-locks/testCompile.lockfile | 19 +- .../testCompileClasspath.lockfile | 19 +- .../dependency-locks/testRuntime.lockfile | 19 +- .../testRuntimeClasspath.lockfile | 19 +- .../gradle/dependency-locks/compile.lockfile | 19 +- .../compileClasspath.lockfile | 19 +- .../gradle/dependency-locks/default.lockfile | 19 +- .../gradle/dependency-locks/runtime.lockfile | 19 +- .../runtimeClasspath.lockfile | 19 +- .../dependency-locks/testCompile.lockfile | 19 +- .../testCompileClasspath.lockfile | 19 +- .../dependency-locks/testRuntime.lockfile | 19 +- .../testRuntimeClasspath.lockfile | 19 +- .../gradle/dependency-locks/compile.lockfile | 19 +- .../compileClasspath.lockfile | 19 +- .../gradle/dependency-locks/default.lockfile | 19 +- .../gradle/dependency-locks/runtime.lockfile | 19 +- .../runtimeClasspath.lockfile | 19 +- .../dependency-locks/testCompile.lockfile | 19 +- .../testCompileClasspath.lockfile | 19 +- .../dependency-locks/testRuntime.lockfile | 19 +- .../testRuntimeClasspath.lockfile | 19 +- util/gradle/dependency-locks/compile.lockfile | 6 +- .../compileClasspath.lockfile | 6 +- util/gradle/dependency-locks/default.lockfile | 6 +- .../dependency-locks/deploy_jar.lockfile | 6 +- util/gradle/dependency-locks/runtime.lockfile | 6 +- .../runtimeClasspath.lockfile | 6 +- .../dependency-locks/testCompile.lockfile | 6 +- .../testCompileClasspath.lockfile | 6 +- .../dependency-locks/testRuntime.lockfile | 6 +- .../testRuntimeClasspath.lockfile | 6 +- 186 files changed, 2415 insertions(+), 1600 deletions(-) delete mode 100644 core/src/main/java/google/registry/gcs/GcsServiceModule.java create mode 100644 core/src/test/java/google/registry/gcs/GcsUtilsTest.java create mode 100644 core/src/test/java/google/registry/gcs/backport/FakeStorageRpc.java create mode 100644 core/src/test/java/google/registry/gcs/backport/LocalStorageHelper.java create mode 100644 core/src/test/java/google/registry/gcs/backport/README.md delete mode 100644 core/src/test/java/google/registry/testing/GcsTestingUtils.java diff --git a/buildSrc/gradle/dependency-locks/compile.lockfile b/buildSrc/gradle/dependency-locks/compile.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/compile.lockfile +++ b/buildSrc/gradle/dependency-locks/compile.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/compileClasspath.lockfile b/buildSrc/gradle/dependency-locks/compileClasspath.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/compileClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/compileClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/deploy_jar.lockfile b/buildSrc/gradle/dependency-locks/deploy_jar.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/deploy_jar.lockfile +++ b/buildSrc/gradle/dependency-locks/deploy_jar.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/runtime.lockfile b/buildSrc/gradle/dependency-locks/runtime.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/runtime.lockfile +++ b/buildSrc/gradle/dependency-locks/runtime.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile b/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile index add05e4df..7b80e344a 100644 --- a/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/runtimeClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 @@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6 org.apache.httpcomponents:httpclient:4.5.13 org.apache.httpcomponents:httpcore:4.4.14 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.json:json:20160212 org.ow2.asm:asm-analysis:7.0 org.ow2.asm:asm-commons:7.0 diff --git a/buildSrc/gradle/dependency-locks/testCompile.lockfile b/buildSrc/gradle/dependency-locks/testCompile.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testCompile.lockfile +++ b/buildSrc/gradle/dependency-locks/testCompile.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile b/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/testCompileClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/buildSrc/gradle/dependency-locks/testRuntime.lockfile b/buildSrc/gradle/dependency-locks/testRuntime.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testRuntime.lockfile +++ b/buildSrc/gradle/dependency-locks/testRuntime.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile b/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile index ddd2b2326..e23623464 100644 --- a/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/buildSrc/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6 com.google.errorprone:error_prone_annotations:2.5.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:1.39.0 com.google.http-client:google-http-client-appengine:1.39.0 diff --git a/common/gradle/dependency-locks/compile.lockfile b/common/gradle/dependency-locks/compile.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/compile.lockfile +++ b/common/gradle/dependency-locks/compile.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/compileClasspath.lockfile b/common/gradle/dependency-locks/compileClasspath.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/compileClasspath.lockfile +++ b/common/gradle/dependency-locks/compileClasspath.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/default.lockfile b/common/gradle/dependency-locks/default.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/default.lockfile +++ b/common/gradle/dependency-locks/default.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/deploy_jar.lockfile b/common/gradle/dependency-locks/deploy_jar.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/deploy_jar.lockfile +++ b/common/gradle/dependency-locks/deploy_jar.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/runtime.lockfile b/common/gradle/dependency-locks/runtime.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/runtime.lockfile +++ b/common/gradle/dependency-locks/runtime.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/runtimeClasspath.lockfile b/common/gradle/dependency-locks/runtimeClasspath.lockfile index 90fc663bc..521b55f9e 100644 --- a/common/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/common/gradle/dependency-locks/runtimeClasspath.lockfile @@ -2,11 +2,11 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. com.google.code.findbugs:jsr305:3.0.2 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 javax.inject:javax.inject:1 joda-time:joda-time:2.9.2 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 diff --git a/common/gradle/dependency-locks/testCompile.lockfile b/common/gradle/dependency-locks/testCompile.lockfile index 017019006..fbea9547f 100644 --- a/common/gradle/dependency-locks/testCompile.lockfile +++ b/common/gradle/dependency-locks/testCompile.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testCompileClasspath.lockfile b/common/gradle/dependency-locks/testCompileClasspath.lockfile index 017019006..fbea9547f 100644 --- a/common/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/common/gradle/dependency-locks/testCompileClasspath.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testRuntime.lockfile b/common/gradle/dependency-locks/testRuntime.lockfile index 6930dcb16..d45bd7305 100644 --- a/common/gradle/dependency-locks/testRuntime.lockfile +++ b/common/gradle/dependency-locks/testRuntime.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testRuntimeClasspath.lockfile b/common/gradle/dependency-locks/testRuntimeClasspath.lockfile index 6930dcb16..d45bd7305 100644 --- a/common/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/common/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingCompile.lockfile b/common/gradle/dependency-locks/testingCompile.lockfile index c67ff6845..1893cd4b9 100644 --- a/common/gradle/dependency-locks/testingCompile.lockfile +++ b/common/gradle/dependency-locks/testingCompile.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingCompileClasspath.lockfile b/common/gradle/dependency-locks/testingCompileClasspath.lockfile index c67ff6845..1893cd4b9 100644 --- a/common/gradle/dependency-locks/testingCompileClasspath.lockfile +++ b/common/gradle/dependency-locks/testingCompileClasspath.lockfile @@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingRuntime.lockfile b/common/gradle/dependency-locks/testingRuntime.lockfile index 7f20d13e3..fdb906898 100644 --- a/common/gradle/dependency-locks/testingRuntime.lockfile +++ b/common/gradle/dependency-locks/testingRuntime.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile b/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile index 7f20d13e3..fdb906898 100644 --- a/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile +++ b/common/gradle/dependency-locks/testingRuntimeClasspath.lockfile @@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.truth:truth:1.1.2 diff --git a/config/presubmits.py b/config/presubmits.py index 442676ea1..9d85c4c3c 100644 --- a/config/presubmits.py +++ b/config/presubmits.py @@ -79,8 +79,8 @@ PRESUBMITS = { r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.", ("java", "js", "soy", "sql", "py", "sh", "gradle"), { ".git", "/build/", "/generated/", "/generated_tests/", - "node_modules/", "JUnitBackports.java", "registrar_bin.", - "registrar_dbg.", "google-java-format-diff.py", + "node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java", + "registrar_bin.", "registrar_dbg.", "google-java-format-diff.py", "nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js" }, REQUIRED): "File did not include the license header.", diff --git a/core/build.gradle b/core/build.gradle index ca7809029..f280b0f99 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -197,6 +197,7 @@ dependencies { compile deps['com.google.apis:google-api-services-groupssettings'] compile deps['com.google.apis:google-api-services-monitoring'] compile deps['com.google.apis:google-api-services-sheets'] + compile deps['com.google.apis:google-api-services-storage'] testCompile deps['com.google.appengine:appengine-api-stubs'] compile deps['com.google.appengine.tools:appengine-gcs-client'] compile deps['com.google.appengine.tools:appengine-mapreduce'] @@ -221,6 +222,8 @@ dependencies { gradleLint.ignore('unused-dependency') { compile deps['com.google.gwt:gwt-user'] } + compile deps['com.google.cloud:google-cloud-core'] + compile deps['com.google.cloud:google-cloud-storage'] compile deps['com.google.http-client:google-http-client'] compile deps['com.google.http-client:google-http-client-appengine'] compile deps['com.google.http-client:google-http-client-jackson2'] @@ -315,6 +318,7 @@ dependencies { annotationProcessor project(':processor') testAnnotationProcessor project(':processor') + testCompile deps['com.google.cloud:google-cloud-nio'] testCompile deps['com.google.appengine:appengine-testing'] testCompile deps['com.google.guava:guava-testlib'] testCompile deps['com.google.monitoring-client:contrib'] diff --git a/core/gradle/dependency-locks/annotationProcessor.lockfile b/core/gradle/dependency-locks/annotationProcessor.lockfile index f0a2ecd5d..09dbb3924 100644 --- a/core/gradle/dependency-locks/annotationProcessor.lockfile +++ b/core/gradle/dependency-locks/annotationProcessor.lockfile @@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33 com.google.dagger:dagger-spi:2.33 com.google.dagger:dagger:2.33 com.google.errorprone:error_prone_annotation:2.3.4 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.errorprone:error_prone_check_api:2.3.4 com.google.errorprone:error_prone_core:2.3.4 com.google.errorprone:error_prone_type_annotations:2.3.4 com.google.errorprone:javac-shaded:9-dev-r4023-3 com.google.googlejavaformat:google-java-format:1.5 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.protobuf:protobuf-java:3.4.0 @@ -32,7 +32,7 @@ javax.inject:javax.inject:1 javax.persistence:javax.persistence-api:2.2 net.ltgt.gradle.incap:incap:0.2 org.checkerframework:checker-compat-qual:2.5.3 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.checkerframework:dataflow:3.0.0 org.checkerframework:javacutil:3.0.0 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20 diff --git a/core/gradle/dependency-locks/compile.lockfile b/core/gradle/dependency-locks/compile.lockfile index e3e31a555..da39c21db 100644 --- a/core/gradle/dependency-locks/compile.lockfile +++ b/core/gradle/dependency-locks/compile.lockfile @@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/compileClasspath.lockfile b/core/gradle/dependency-locks/compileClasspath.lockfile index 80864edca..1631f4d24 100644 --- a/core/gradle/dependency-locks/compileClasspath.lockfile +++ b/core/gradle/dependency-locks/compileClasspath.lockfile @@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.14.0 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -215,7 +216,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.conscrypt:conscrypt-openjdk-uber:2.5.1 diff --git a/core/gradle/dependency-locks/default.lockfile b/core/gradle/dependency-locks/default.lockfile index 350bc7d70..d8072a9ef 100644 --- a/core/gradle/dependency-locks/default.lockfile +++ b/core/gradle/dependency-locks/default.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/deploy_jar.lockfile b/core/gradle/dependency-locks/deploy_jar.lockfile index d67debded..c3b897de3 100644 --- a/core/gradle/dependency-locks/deploy_jar.lockfile +++ b/core/gradle/dependency-locks/deploy_jar.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/nonprodCompile.lockfile b/core/gradle/dependency-locks/nonprodCompile.lockfile index e3e31a555..da39c21db 100644 --- a/core/gradle/dependency-locks/nonprodCompile.lockfile +++ b/core/gradle/dependency-locks/nonprodCompile.lockfile @@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile b/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile index 70618ba77..c28288d48 100644 --- a/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile +++ b/core/gradle/dependency-locks/nonprodCompileClasspath.lockfile @@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.14.0 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -216,7 +217,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.conscrypt:conscrypt-openjdk-uber:2.5.1 diff --git a/core/gradle/dependency-locks/nonprodRuntime.lockfile b/core/gradle/dependency-locks/nonprodRuntime.lockfile index 36adf8801..42f0cec4d 100644 --- a/core/gradle/dependency-locks/nonprodRuntime.lockfile +++ b/core/gradle/dependency-locks/nonprodRuntime.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile b/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile index 36adf8801..42f0cec4d 100644 --- a/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile +++ b/core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/runtime.lockfile b/core/gradle/dependency-locks/runtime.lockfile index 36adf8801..42f0cec4d 100644 --- a/core/gradle/dependency-locks/runtime.lockfile +++ b/core/gradle/dependency-locks/runtime.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/runtimeClasspath.lockfile b/core/gradle/dependency-locks/runtimeClasspath.lockfile index d67debded..c3b897de3 100644 --- a/core/gradle/dependency-locks/runtimeClasspath.lockfile +++ b/core/gradle/dependency-locks/runtimeClasspath.lockfile @@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 +com.google.api.grpc:proto-google-iam-v1:1.0.10 com.google.api:api-common:1.10.1 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 +com.google.api:gax-httpjson:0.79.0 com.google.api:gax:1.62.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.1 +com.google.cloud:google-cloud-core:1.94.3 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.113.12 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 @@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 com.google.http-client:google-http-client-apache-v2:1.39.0 @@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.protobuf:protobuf-java-util:3.15.3 +com.google.protobuf:protobuf-java:3.15.3 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.googlecode.charts4j:charts4j:1.3 @@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61 org.bouncycastle:bcpkix-jdk15on:1.61 org.bouncycastle:bcprov-jdk15on:1.61 org.checkerframework:checker-compat-qual:2.5.5 -org.checkerframework:checker-qual:3.7.0 +org.checkerframework:checker-qual:3.8.0 org.codehaus.jackson:jackson-core-asl:1.9.13 org.codehaus.jackson:jackson-mapper-asl:1.9.13 org.codehaus.mojo:animal-sniffer-annotations:1.20 diff --git a/core/gradle/dependency-locks/testAnnotationProcessor.lockfile b/core/gradle/dependency-locks/testAnnotationProcessor.lockfile index 3cb0bf2ea..69c9128fc 100644 --- a/core/gradle/dependency-locks/testAnnotationProcessor.lockfile +++ b/core/gradle/dependency-locks/testAnnotationProcessor.lockfile @@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33 com.google.dagger:dagger-spi:2.33 com.google.dagger:dagger:2.33 com.google.errorprone:error_prone_annotation:2.3.4 -com.google.errorprone:error_prone_annotations:2.3.4 +com.google.errorprone:error_prone_annotations:2.5.1 com.google.errorprone:error_prone_check_api:2.3.4 com.google.errorprone:error_prone_core:2.3.4 com.google.errorprone:error_prone_type_annotations:2.3.4 com.google.errorprone:javac-shaded:9-dev-r4023-3 com.google.googlejavaformat:google-java-format:1.5 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava:30.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.j2objc:j2objc-annotations:1.3 com.google.protobuf:protobuf-java:3.4.0 @@ -30,7 +30,7 @@ javax.inject:javax.inject:1 javax.persistence:javax.persistence-api:2.2 net.ltgt.gradle.incap:incap:0.2 org.checkerframework:checker-compat-qual:2.5.3 -org.checkerframework:checker-qual:3.5.0 +org.checkerframework:checker-qual:3.8.0 org.checkerframework:dataflow:3.0.0 org.checkerframework:javacutil:3.0.0 org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20 diff --git a/core/gradle/dependency-locks/testCompile.lockfile b/core/gradle/dependency-locks/testCompile.lockfile index 8e64dd0b4..fbbece5be 100644 --- a/core/gradle/dependency-locks/testCompile.lockfile +++ b/core/gradle/dependency-locks/testCompile.lockfile @@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0 args4j:args4j:2.0.23 cglib:cglib-nodep:2.2 com.beust:jcommander:1.60 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -54,12 +54,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -84,10 +84,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -98,32 +98,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -135,9 +137,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -161,7 +163,7 @@ io.github.classgraph:classgraph:4.8.102 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -199,7 +201,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 net.java.dev.jna:jna:5.5.0 @@ -307,7 +309,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.xerial.snappy:snappy-java:1.1.4 diff --git a/core/gradle/dependency-locks/testCompileClasspath.lockfile b/core/gradle/dependency-locks/testCompileClasspath.lockfile index d112fcc04..270390b42 100644 --- a/core/gradle/dependency-locks/testCompileClasspath.lockfile +++ b/core/gradle/dependency-locks/testCompileClasspath.lockfile @@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0 args4j:args4j:2.0.23 cglib:cglib-nodep:2.2 com.beust:jcommander:1.60 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -53,12 +53,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -83,10 +83,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -97,31 +97,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -133,9 +135,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.14.0 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -159,7 +161,7 @@ io.github.classgraph:classgraph:4.8.102 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -194,7 +196,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 net.java.dev.jna:jna:5.5.0 @@ -301,7 +303,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.xerial.snappy:snappy-java:1.1.4 diff --git a/core/gradle/dependency-locks/testRuntime.lockfile b/core/gradle/dependency-locks/testRuntime.lockfile index ca0377997..5096bf38c 100644 --- a/core/gradle/dependency-locks/testRuntime.lockfile +++ b/core/gradle/dependency-locks/testRuntime.lockfile @@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.arnx:nashorn-promise:0.1.1 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 @@ -320,7 +322,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.webjars.npm:viz.js-for-graphviz-java:2.1.3 diff --git a/core/gradle/dependency-locks/testRuntimeClasspath.lockfile b/core/gradle/dependency-locks/testRuntimeClasspath.lockfile index e78acec00..d9d835407 100644 --- a/core/gradle/dependency-locks/testRuntimeClasspath.lockfile +++ b/core/gradle/dependency-locks/testRuntimeClasspath.lockfile @@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0 com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0 -com.fasterxml.jackson.core:jackson-annotations:2.12.1 -com.fasterxml.jackson.core:jackson-core:2.12.1 -com.fasterxml.jackson.core:jackson-databind:2.12.1 -com.fasterxml.jackson:jackson-bom:2.12.1 +com.fasterxml.jackson.core:jackson-annotations:2.12.3 +com.fasterxml.jackson.core:jackson-core:2.12.3 +com.fasterxml.jackson.core:jackson-databind:2.12.3 +com.fasterxml.jackson:jackson-bom:2.12.3 com.fasterxml:classmate:1.5.1 com.github.docker-java:docker-java-api:3.2.7 com.github.docker-java:docker-java-transport-zerodep:3.2.7 @@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3 com.google.api-client:google-api-client-jackson2:1.30.10 com.google.api-client:google-api-client-java6:1.31.3 com.google.api-client:google-api-client-servlet:1.31.3 -com.google.api-client:google-api-client:1.31.3 +com.google.api-client:google-api-client:1.31.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5 com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5 @@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0 com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2 com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2 -com.google.api.grpc:proto-google-common-protos:2.1.0 -com.google.api.grpc:proto-google-iam-v1:1.0.9 -com.google.api:api-common:1.10.1 +com.google.api.grpc:proto-google-common-protos:2.3.2 +com.google.api.grpc:proto-google-iam-v1:1.0.14 +com.google.api:api-common:1.10.3 com.google.api:gax-grpc:1.62.0 -com.google.api:gax-httpjson:0.76.1 -com.google.api:gax:1.62.0 +com.google.api:gax-httpjson:0.81.0 +com.google.api:gax:1.65.0 com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0 com.google.apis:google-api-services-appengine:v1-rev130-1.25.0 com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10 @@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0 com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10 com.google.apis:google-api-services-sheets:v4-rev612-1.25.0 com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0 -com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10 +com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0 com.google.appengine.tools:appengine-gcs-client:0.8.1 com.google.appengine.tools:appengine-mapreduce:0.9 com.google.appengine.tools:appengine-pipeline:0.2.13 @@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86 com.google.appengine:appengine-api-stubs:1.9.86 com.google.appengine:appengine-remote-api:1.9.86 com.google.appengine:appengine-testing:1.9.86 -com.google.auth:google-auth-library-credentials:0.24.1 -com.google.auth:google-auth-library-oauth2-http:0.24.1 +com.google.auth:google-auth-library-credentials:0.26.0 +com.google.auth:google-auth-library-oauth2-http:0.26.0 com.google.auto.service:auto-service-annotations:1.0-rc7 -com.google.auto.value:auto-value-annotations:1.7.4 +com.google.auto.value:auto-value-annotations:1.8.1 com.google.auto.value:auto-value:1.7.4 com.google.cloud.bigdataoss:gcsio:2.1.6 com.google.cloud.bigdataoss:util:2.1.6 @@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2 com.google.cloud:google-cloud-bigquerystorage:1.5.5 com.google.cloud:google-cloud-bigtable:1.14.0 com.google.cloud:google-cloud-core-grpc:1.93.9 -com.google.cloud:google-cloud-core-http:1.93.9 -com.google.cloud:google-cloud-core:1.93.9 +com.google.cloud:google-cloud-core-http:1.94.8 +com.google.cloud:google-cloud-core:1.95.0 +com.google.cloud:google-cloud-nio:0.123.2 com.google.cloud:google-cloud-pubsub:1.110.0 com.google.cloud:google-cloud-pubsublite:0.7.0 com.google.cloud:google-cloud-secretmanager:1.4.0 com.google.cloud:google-cloud-spanner:2.0.2 +com.google.cloud:google-cloud-storage:1.115.0 com.google.code.findbugs:jsr305:3.0.2 com.google.code.gson:gson:2.8.6 com.google.common.html.types:types:1.0.6 com.google.dagger:dagger:2.33 -com.google.errorprone:error_prone_annotations:2.5.1 +com.google.errorprone:error_prone_annotations:2.7.1 com.google.escapevelocity:escapevelocity:0.9.1 com.google.flogger:flogger-system-backend:0.5.1 com.google.flogger:flogger:0.5.1 com.google.flogger:google-extensions:0.5.1 com.google.guava:failureaccess:1.0.1 -com.google.guava:guava-testlib:30.1-jre -com.google.guava:guava:30.1-jre +com.google.guava:guava-testlib:30.1.1-jre +com.google.guava:guava:30.1.1-jre com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.gwt:gwt-user:2.9.0 -com.google.http-client:google-http-client-apache-v2:1.39.0 -com.google.http-client:google-http-client-appengine:1.39.0 -com.google.http-client:google-http-client-gson:1.39.0 -com.google.http-client:google-http-client-jackson2:1.39.0 +com.google.http-client:google-http-client-apache-v2:1.39.2 +com.google.http-client:google-http-client-appengine:1.39.2 +com.google.http-client:google-http-client-gson:1.39.2 +com.google.http-client:google-http-client-jackson2:1.39.2 com.google.http-client:google-http-client-protobuf:1.33.0 -com.google.http-client:google-http-client:1.39.0 +com.google.http-client:google-http-client:1.39.2 com.google.inject.extensions:guice-multibindings:4.1.0 com.google.inject:guice:4.1.0 com.google.j2objc:j2objc-annotations:1.3 @@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4 com.google.oauth-client:google-oauth-client-java6:1.31.4 com.google.oauth-client:google-oauth-client-jetty:1.31.4 com.google.oauth-client:google-oauth-client-servlet:1.31.4 -com.google.oauth-client:google-oauth-client:1.31.4 -com.google.protobuf:protobuf-java-util:3.15.2 -com.google.protobuf:protobuf-java:3.15.2 +com.google.oauth-client:google-oauth-client:1.31.5 +com.google.protobuf:protobuf-java-util:3.17.2 +com.google.protobuf:protobuf-java:3.17.2 com.google.re2j:re2j:1.6 com.google.template:soy:2021-02-01 com.google.truth.extensions:truth-java8-extension:1.1.2 @@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9 io.grpc:grpc-alts:1.36.0 io.grpc:grpc-api:1.36.0 io.grpc:grpc-auth:1.36.0 -io.grpc:grpc-context:1.36.0 +io.grpc:grpc-context:1.37.1 io.grpc:grpc-core:1.36.0 io.grpc:grpc-grpclb:1.36.0 io.grpc:grpc-netty-shaded:1.36.0 @@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA javax.xml.bind:jaxb-api:2.3.1 jline:jline:1.0 joda-time:joda-time:2.10.5 -junit:junit:4.13.1 +junit:junit:4.13.2 net.arnx:nashorn-promise:0.1.1 net.bytebuddy:byte-buddy-agent:1.10.19 net.bytebuddy:byte-buddy:1.10.19 @@ -321,7 +323,7 @@ org.testcontainers:junit-jupiter:1.15.2 org.testcontainers:postgresql:1.15.2 org.testcontainers:selenium:1.15.2 org.testcontainers:testcontainers:1.15.2 -org.threeten:threetenbp:1.5.0 +org.threeten:threetenbp:1.5.1 org.tukaani:xz:1.5 org.w3c.css:sac:1.3 org.webjars.npm:viz.js-for-graphviz-java:2.1.3 diff --git a/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java b/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java index c23d59372..767326880 100644 --- a/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java +++ b/core/src/main/java/google/registry/backup/ExportCommitLogDiffAction.java @@ -28,18 +28,17 @@ import static google.registry.model.ofy.CommitLogBucket.getBucketKey; import static google.registry.model.ofy.ObjectifyService.auditedOfy; import static google.registry.util.DateTimeUtils.START_OF_TIME; import static google.registry.util.DateTimeUtils.isAtOrAfter; -import static java.nio.channels.Channels.newOutputStream; import static java.util.Comparator.comparingLong; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; import com.googlecode.objectify.Key; import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; import google.registry.model.ImmutableObject; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; @@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable { public static final String DIFF_FILE_PREFIX = "commit_diff_until_"; - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; + @Inject @Config("commitLogGcsBucket") String gcsBucket; @Inject @Config("commitLogDiffExportBatchSize") int batchSize; @Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime; @@ -102,13 +102,13 @@ public final class ExportCommitLogDiffAction implements Runnable { List> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint); logger.atInfo().log("Found %d manifests to export", sortedKeys.size()); // Open an output channel to GCS, wrapped in a stream for convenience. - try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace( - new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString()) - .addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString()) - .addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())) - .build()))) { + try (OutputStream gcsStream = + gcsUtils.openOutputStream( + BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime), + ImmutableMap.of( + LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(), + UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(), + NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) { // Export the upper checkpoint itself. serializeEntity(upperCheckpoint, gcsStream); // If there are no manifests to export, stop early, now that we've written out the file with diff --git a/core/src/main/java/google/registry/backup/GcsDiffFileLister.java b/core/src/main/java/google/registry/backup/GcsDiffFileLister.java index 7fd887a8e..19305a9e2 100644 --- a/core/src/main/java/google/registry/backup/GcsDiffFileLister.java +++ b/core/src/main/java/google/registry/backup/GcsDiffFileLister.java @@ -21,19 +21,16 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME; import static google.registry.util.DateTimeUtils.isBeforeOrAt; import static google.registry.util.DateTimeUtils.latestOf; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.ListItem; -import com.google.appengine.tools.cloudstorage.ListOptions; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import google.registry.backup.BackupModule.Backups; +import google.registry.gcs.GcsUtils; import java.io.IOException; -import java.util.Iterator; import java.util.Map; import java.util.TreeMap; import javax.annotation.Nullable; @@ -45,7 +42,8 @@ class GcsDiffFileLister { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; + @Inject @Backups ListeningExecutorService executor; @Inject GcsDiffFileLister() {} @@ -56,22 +54,22 @@ class GcsDiffFileLister { */ private boolean constructDiffSequence( String gcsBucket, - Map> upperBoundTimesToMetadata, + Map> upperBoundTimesToBlobInfo, DateTime fromTime, DateTime lastTime, - TreeMap sequence) { + TreeMap sequence) { DateTime checkpointTime = lastTime; while (isBeforeOrAt(fromTime, checkpointTime)) { - GcsFileMetadata metadata; - if (upperBoundTimesToMetadata.containsKey(checkpointTime)) { - metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime)); + BlobInfo blobInfo; + if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) { + blobInfo = Futures.getUnchecked(upperBoundTimesToBlobInfo.get(checkpointTime)); } else { String filename = DIFF_FILE_PREFIX + checkpointTime; logger.atInfo().log("Patching GCS list; discovered file: %s", filename); - metadata = getMetadata(gcsBucket, filename); + blobInfo = getBlobInfo(gcsBucket, filename); // If we hit a gap, quit. - if (metadata == null) { + if (blobInfo == null) { logger.atInfo().log( "Gap discovered in sequence terminating at %s, missing file: %s", sequence.lastKey(), filename); @@ -79,14 +77,14 @@ class GcsDiffFileLister { return false; } } - sequence.put(checkpointTime, metadata); - checkpointTime = getLowerBoundTime(metadata); + sequence.put(checkpointTime, blobInfo); + checkpointTime = getLowerBoundTime(blobInfo); } logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime); return true; } - ImmutableList listDiffFiles( + ImmutableList listDiffFiles( String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) { logger.atInfo().log("Requested restore from time: %s", fromTime); if (toTime != null) { @@ -95,28 +93,25 @@ class GcsDiffFileLister { // List all of the diff files on GCS and build a map from each file's upper checkpoint time // (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with // an upper checkpoint time > fromTime. - TreeMap> upperBoundTimesToMetadata - = new TreeMap<>(); - Iterator listItems; + TreeMap> upperBoundTimesToBlobInfo = new TreeMap<>(); + ImmutableList strippedFilenames; try { - // TODO(b/23554360): Use a smarter prefixing strategy to speed this up. - listItems = gcsService.list( - gcsBucket, - new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build()); + strippedFilenames = gcsUtils.listFolderObjects(gcsBucket, DIFF_FILE_PREFIX); } catch (IOException e) { throw new RuntimeException(e); } DateTime lastUpperBoundTime = START_OF_TIME; - while (listItems.hasNext()) { - final String filename = listItems.next().getName(); - DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length())); + + for (String strippedFilename : strippedFilenames) { + final String filename = DIFF_FILE_PREFIX + strippedFilename; + DateTime upperBoundTime = DateTime.parse(strippedFilename); if (isInRange(upperBoundTime, fromTime, toTime)) { - upperBoundTimesToMetadata.put( - upperBoundTime, executor.submit(() -> getMetadata(gcsBucket, filename))); + upperBoundTimesToBlobInfo.put( + upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename))); lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime); } } - if (upperBoundTimesToMetadata.isEmpty()) { + if (upperBoundTimesToBlobInfo.isEmpty()) { logger.atInfo().log("No files found"); return ImmutableList.of(); } @@ -129,23 +124,23 @@ class GcsDiffFileLister { // metadata of a file is sufficient to identify the preceding file, so if we start from the // last file and work backwards we can verify that we have no holes in our chain (although we // may be missing files at the end). - TreeMap sequence = new TreeMap<>(); + TreeMap sequence = new TreeMap<>(); logger.atInfo().log("Restoring until: %s", lastUpperBoundTime); boolean inconsistentFileSet = !constructDiffSequence( - gcsBucket, upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence); + gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence); // Verify that all of the elements in the original set are represented in the sequence. If we // find anything that's not represented, construct a sequence for it. boolean checkForMoreExtraDiffs = true; // Always loop at least once. while (checkForMoreExtraDiffs) { checkForMoreExtraDiffs = false; - for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) { + for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) { if (!isInRange(key, fromTime, toTime)) { break; } if (!sequence.containsKey(key)) { - constructDiffSequence(gcsBucket, upperBoundTimesToMetadata, fromTime, key, sequence); + constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence); checkForMoreExtraDiffs = true; inconsistentFileSet = true; break; @@ -173,15 +168,11 @@ class GcsDiffFileLister { return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end)); } - private DateTime getLowerBoundTime(GcsFileMetadata metadata) { - return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT)); + private DateTime getLowerBoundTime(BlobInfo blobInfo) { + return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT)); } - private GcsFileMetadata getMetadata(String gcsBucket, String filename) { - try { - return gcsService.getMetadata(new GcsFilename(gcsBucket, filename)); - } catch (IOException e) { - throw new RuntimeException(e); - } + private BlobInfo getBlobInfo(String gcsBucket, String filename) { + return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename)); } } diff --git a/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java b/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java index 5709ab07e..8bcb1c0c3 100644 --- a/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java +++ b/core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java @@ -25,12 +25,12 @@ import static org.joda.time.Duration.standardHours; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.Key; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobInfo; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; import google.registry.model.common.DatabaseMigrationStateSchedule; import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection; @@ -51,7 +51,6 @@ import google.registry.util.Clock; import google.registry.util.RequestStatusChecker; import java.io.IOException; import java.io.InputStream; -import java.nio.channels.Channels; import java.util.Optional; import javax.inject.Inject; import javax.servlet.http.HttpServletResponse; @@ -70,14 +69,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable { static final String PATH = "/_dr/task/replayCommitLogsToSql"; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final int BLOCK_SIZE = - 1024 * 1024; // Buffer 1mb at a time, for no particular reason. + private static final Duration LEASE_LENGTH = standardHours(1); // Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE // request timeouts private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5); - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; @Inject Response response; @Inject RequestStatusChecker requestStatusChecker; @Inject GcsDiffFileLister diffLister; @@ -122,13 +120,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable { } try { logger.atInfo().log("Beginning replay of commit logs."); - ImmutableList commitLogFiles = getFilesToReplay(); + ImmutableList commitLogFiles = getFilesToReplay(); if (dryRun) { response.setStatus(HttpServletResponse.SC_OK); ImmutableList filenames = commitLogFiles.stream() .limit(10) - .map(file -> file.getFilename().getObjectName()) + .map(file -> file.getName()) .collect(toImmutableList()); String dryRunMessage = "Running in dry-run mode; would have processed %d files. They are (limit 10):\n" @@ -152,22 +150,22 @@ public class ReplayCommitLogsToSqlAction implements Runnable { } } - private ImmutableList getFilesToReplay() { + private ImmutableList getFilesToReplay() { // Start at the first millisecond we haven't seen yet DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1)); logger.atInfo().log("Starting replay from: %s.", fromTime); // If there's an inconsistent file set, this will throw IllegalStateException and the job // will try later -- this is likely because an export hasn't finished yet. - ImmutableList commitLogFiles = + ImmutableList commitLogFiles = diffLister.listDiffFiles(gcsBucket, fromTime, /* current time */ null); logger.atInfo().log("Found %d new commit log files to process.", commitLogFiles.size()); return commitLogFiles; } - private void replayFiles(ImmutableList commitLogFiles) { + private void replayFiles(ImmutableList commitLogFiles) { DateTime replayTimeoutTime = clock.nowUtc().plus(REPLAY_TIMEOUT_DURATION); int processedFiles = 0; - for (GcsFileMetadata metadata : commitLogFiles) { + for (BlobInfo metadata : commitLogFiles) { // One transaction per GCS file jpaTm().transact(() -> processFile(metadata)); processedFiles++; @@ -181,31 +179,26 @@ public class ReplayCommitLogsToSqlAction implements Runnable { logger.atInfo().log("Replayed %d commit log files to SQL successfully.", processedFiles); } - private void processFile(GcsFileMetadata metadata) { + private void processFile(BlobInfo metadata) { logger.atInfo().log( - "Processing commit log file %s of size %d B.", - metadata.getFilename(), metadata.getLength()); - try (InputStream input = - Channels.newInputStream( - gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) { + "Processing commit log file %s of size %d B.", metadata.getName(), metadata.getSize()); + try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) { // Load and process the Datastore transactions one at a time ImmutableList> allTransactions = CommitLogImports.loadEntitiesByTransaction(input); logger.atInfo().log( "Replaying %d transactions from commit log file %s.", - allTransactions.size(), metadata.getFilename()); + allTransactions.size(), metadata.getName()); allTransactions.forEach(this::replayTransaction); // if we succeeded, set the last-seen time - DateTime checkpoint = - DateTime.parse( - metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length())); + DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length())); SqlReplayCheckpoint.set(checkpoint); logger.atInfo().log( "Replayed %d transactions from commit log file %s.", - allTransactions.size(), metadata.getFilename()); + allTransactions.size(), metadata.getName()); } catch (IOException e) { throw new RuntimeException( - "Errored out while replaying commit log file " + metadata.getFilename(), e); + "Errored out while replaying commit log file " + metadata.getName(), e); } } diff --git a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java index fdf99f69f..be8130692 100644 --- a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java +++ b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java @@ -23,8 +23,7 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy; import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.EntityTranslator; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobInfo; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -36,6 +35,7 @@ import com.googlecode.objectify.Result; import com.googlecode.objectify.util.ResultNow; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; +import google.registry.gcs.GcsUtils; import google.registry.model.ImmutableObject; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; @@ -48,7 +48,6 @@ import google.registry.request.auth.Auth; import google.registry.util.Retrier; import java.io.IOException; import java.io.InputStream; -import java.nio.channels.Channels; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -69,8 +68,6 @@ public class RestoreCommitLogsAction implements Runnable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - static final int BLOCK_SIZE = 1024 * 1024; // Buffer 1mb at a time, for no particular reason. - public static final String PATH = "/_dr/task/restoreCommitLogs"; static final String DRY_RUN_PARAM = "dryRun"; static final String FROM_TIME_PARAM = "fromTime"; @@ -80,7 +77,8 @@ public class RestoreCommitLogsAction implements Runnable { private static final ImmutableSet FORBIDDEN_ENVIRONMENTS = ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX); - @Inject GcsService gcsService; + @Inject GcsUtils gcsUtils; + @Inject @Parameter(DRY_RUN_PARAM) boolean dryRun; @Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime; @Inject @Parameter(TO_TIME_PARAM) DateTime toTime; @@ -109,17 +107,16 @@ public class RestoreCommitLogsAction implements Runnable { } String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket); logger.atInfo().log("Restoring from %s.", gcsBucket); - List diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime); + List diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime); if (diffFiles.isEmpty()) { logger.atInfo().log("Nothing to restore"); return; } Map bucketTimestamps = new HashMap<>(); CommitLogCheckpoint lastCheckpoint = null; - for (GcsFileMetadata metadata : diffFiles) { - logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName()); - try (InputStream input = Channels.newInputStream( - gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) { + for (BlobInfo metadata : diffFiles) { + logger.atInfo().log("Restoring: %s", metadata.getName()); + try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) { PeekingIterator commitLogs = peekingIterator(createDeserializingIterator(input, true)); lastCheckpoint = (CommitLogCheckpoint) commitLogs.next(); diff --git a/core/src/main/java/google/registry/config/RegistryConfig.java b/core/src/main/java/google/registry/config/RegistryConfig.java index e95ce1bc0..b8de3f0d8 100644 --- a/core/src/main/java/google/registry/config/RegistryConfig.java +++ b/core/src/main/java/google/registry/config/RegistryConfig.java @@ -416,17 +416,6 @@ public final class RegistryConfig { return Optional.ofNullable(config.cloudDns.servicePath); } - /** - * Returns size of Google Cloud Storage client connection buffer in bytes. - * - * @see google.registry.gcs.GcsUtils - */ - @Provides - @Config("gcsBufferSize") - public static int provideGcsBufferSize() { - return 1024 * 1024; - } - /** * Returns the email address of the admin account on the G Suite app used to perform * administrative actions. diff --git a/core/src/main/java/google/registry/export/ExportDomainListsAction.java b/core/src/main/java/google/registry/export/ExportDomainListsAction.java index 6b69fdb83..9774323af 100644 --- a/core/src/main/java/google/registry/export/ExportDomainListsAction.java +++ b/core/src/main/java/google/registry/export/ExportDomainListsAction.java @@ -14,7 +14,6 @@ package google.registry.export; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.base.Verify.verifyNotNull; import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; import static google.registry.model.EppResourceUtils.isActive; @@ -24,11 +23,10 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.request.Action.Method.POST; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.RetryParams; import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Suppliers; @@ -79,9 +77,9 @@ public class ExportDomainListsAction implements Runnable { @Inject Response response; @Inject Clock clock; @Inject DriveConnection driveConnection; + @Inject GcsUtils gcsUtils; @Inject @Config("domainListsGcsBucket") String gcsBucket; - @Inject @Config("gcsBufferSize") int gcsBufferSize; @Inject ExportDomainListsAction() {} @Override @@ -95,7 +93,7 @@ public class ExportDomainListsAction implements Runnable { .setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS)) .runMapreduce( new ExportDomainListsMapper(clock.nowUtc(), realTlds), - new ExportDomainListsReducer(gcsBucket, gcsBufferSize), + new ExportDomainListsReducer(gcsBucket, gcsUtils), ImmutableList.of(createEntityInput(DomainBase.class))) .sendLinkToMapreduceConsole(response); } else { @@ -134,7 +132,7 @@ public class ExportDomainListsAction implements Runnable { String domainsList = Joiner.on("\n").join(domains); logger.atInfo().log( "Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld); - exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize); + exportToGcs(tld, domainsList, gcsBucket, gcsUtils); exportToDrive(tld, domainsList, driveConnection); }); } @@ -168,11 +166,9 @@ public class ExportDomainListsAction implements Runnable { } protected static boolean exportToGcs( - String tld, String domains, String gcsBucket, int gcsBufferSize) { - GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt"); - GcsUtils cloudStorage = - new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize); - try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename); + String tld, String domains, String gcsBucket, GcsUtils gcsUtils) { + BlobId blobId = BlobId.of(gcsBucket, tld + ".txt"); + try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId); Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) { osWriter.write(domains); } catch (Throwable e) { @@ -214,7 +210,7 @@ public class ExportDomainListsAction implements Runnable { Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection()); private final String gcsBucket; - private final int gcsBufferSize; + private final GcsUtils gcsUtils; /** * Non-serializable {@link DriveConnection} that will be created when an instance of {@link @@ -224,9 +220,9 @@ public class ExportDomainListsAction implements Runnable { */ private transient DriveConnection driveConnection; - public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) { + public ExportDomainListsReducer(String gcsBucket, GcsUtils gcsUtils) { this.gcsBucket = gcsBucket; - this.gcsBufferSize = gcsBufferSize; + this.gcsUtils = gcsUtils; } @SuppressWarnings("unused") @@ -240,7 +236,7 @@ public class ExportDomainListsAction implements Runnable { ImmutableList domains = ImmutableList.sortedCopyOf(() -> fqdns); String domainsList = Joiner.on('\n').join(domains); logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld); - if (exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize)) { + if (exportToGcs(tld, domainsList, gcsBucket, gcsUtils)) { getContext().incrementCounter("domain lists successful written out to GCS"); } else { getContext().incrementCounter("domain lists failed to write out to GCS"); diff --git a/core/src/main/java/google/registry/gcs/GcsServiceModule.java b/core/src/main/java/google/registry/gcs/GcsServiceModule.java deleted file mode 100644 index 84dd41b30..000000000 --- a/core/src/main/java/google/registry/gcs/GcsServiceModule.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.gcs; - -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; -import com.google.appengine.tools.cloudstorage.RetryParams; -import dagger.Module; -import dagger.Provides; - -/** Dagger module for {@link GcsService}. */ -@Module -public final class GcsServiceModule { - - private static final GcsService gcsService = - GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance()); - - @Provides - static GcsService provideGcsService() { - return gcsService; - } -} diff --git a/core/src/main/java/google/registry/gcs/GcsUtils.java b/core/src/main/java/google/registry/gcs/GcsUtils.java index a159bfb17..7b4ee872c 100644 --- a/core/src/main/java/google/registry/gcs/GcsUtils.java +++ b/core/src/main/java/google/registry/gcs/GcsUtils.java @@ -14,30 +14,39 @@ package google.registry.gcs; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getLast; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.ListOptions; -import com.google.appengine.tools.cloudstorage.ListResult; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobListOption; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; -import google.registry.config.RegistryConfig.Config; +import google.registry.config.CredentialModule.DefaultCredential; +import google.registry.util.GoogleCredentialsBundle; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.nio.ByteBuffer; +import java.io.Serializable; import java.nio.channels.Channels; import javax.annotation.CheckReturnValue; import javax.inject.Inject; -/** Utilities for working with Google Cloud Storage. */ -public class GcsUtils { +/** + * Utilities for working with Google Cloud Storage. + * + *

It is {@link Serializable} so that it can be used in MapReduce or Beam. + */ +public class GcsUtils implements Serializable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -48,30 +57,65 @@ public class GcsUtils { .put("json", MediaType.JSON_UTF_8) .build(); - private final GcsService gcsService; - private final int bufferSize; + private final StorageOptions storageOptions; - @Inject - public GcsUtils(GcsService gcsService, @Config("gcsBufferSize") int bufferSize) { - this.gcsService = gcsService; - this.bufferSize = bufferSize; + private Storage storage() { + return storageOptions.getService(); } - /** Opens a GCS file for reading as an {@link InputStream} with prefetching. */ + @Inject + public GcsUtils(@DefaultCredential GoogleCredentialsBundle credentialsBundle) { + this( + StorageOptions.newBuilder() + .setCredentials(credentialsBundle.getGoogleCredentials()) + .build()); + } + + @VisibleForTesting + public GcsUtils(StorageOptions storageOptions) { + this.storageOptions = storageOptions; + } + + /** Opens a GCS file for reading as an {@link InputStream}. */ @CheckReturnValue - public InputStream openInputStream(GcsFilename filename) { - return Channels.newInputStream(gcsService.openPrefetchingReadChannel(filename, 0, bufferSize)); + public InputStream openInputStream(BlobId blobId) { + return Channels.newInputStream(storage().reader(blobId)); } /** Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files. */ @CheckReturnValue - public OutputStream openOutputStream(GcsFilename filename) throws IOException { - return Channels.newOutputStream(gcsService.createOrReplace(filename, getOptions(filename))); + public OutputStream openOutputStream(BlobId blobId) { + return Channels.newOutputStream(storage().writer(createBlobInfo(blobId))); + } + + /** + * Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files and setting + * the given metadata. + */ + @CheckReturnValue + public OutputStream openOutputStream(BlobId blobId, ImmutableMap metadata) { + return Channels.newOutputStream( + storage().writer(BlobInfo.newBuilder(blobId).setMetadata(metadata).build())); } /** Creates a GCS file with the given byte contents, overwriting existing files. */ - public void createFromBytes(GcsFilename filename, byte[] bytes) throws IOException { - gcsService.createOrReplace(filename, getOptions(filename), ByteBuffer.wrap(bytes)); + public void createFromBytes(BlobId blobId, byte[] bytes) throws StorageException { + createFromBytes(createBlobInfo(blobId), bytes); + } + + /** Creates a GCS file with the given byte contents and metadata, overwriting existing files. */ + public void createFromBytes(BlobInfo blobInfo, byte[] bytes) throws StorageException { + storage().create(blobInfo, bytes); + } + + /** Read the content of the given GCS file and return it in a byte array. */ + public byte[] readBytesFrom(BlobId blobId) throws StorageException { + return storage().readAllBytes(blobId); + } + + /** Delete the given GCS file. */ + public void delete(BlobId blobId) throws StorageException { + storage().delete(blobId); } /** @@ -86,37 +130,55 @@ public class GcsUtils { */ public ImmutableList listFolderObjects(String bucketName, String prefix) throws IOException { - ListResult result = - gcsService.list(bucketName, new ListOptions.Builder().setPrefix(prefix).build()); - final ImmutableList.Builder builder = new ImmutableList.Builder<>(); - result.forEachRemaining( - listItem -> { - if (!listItem.isDirectory()) { - builder.add(listItem.getName().replaceFirst(prefix, "")); - } - }); - return builder.build(); + return Streams.stream(storage().list(bucketName, BlobListOption.prefix(prefix)).iterateAll()) + .map(blob -> blob.getName().substring(prefix.length())) + .collect(toImmutableList()); } /** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */ - public boolean existsAndNotEmpty(GcsFilename file) { - GcsFileMetadata metadata; + public boolean existsAndNotEmpty(BlobId blobId) { try { - metadata = gcsService.getMetadata(file); - } catch (IOException e) { + Blob blob = storage().get(blobId); + return blob != null && blob.getSize() > 0; + } catch (StorageException e) { logger.atWarning().withCause(e).log("Failed to check if GCS file exists"); return false; } - return metadata != null && metadata.getLength() > 0; } - /** Determines most appropriate {@link GcsFileOptions} based on filename extension. */ - private static GcsFileOptions getOptions(GcsFilename filename) { - GcsFileOptions.Builder builder = new GcsFileOptions.Builder().cacheControl("no-cache"); - MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(filename.getObjectName()))); + /** Returns the user defined metadata of a GCS file if the file exists, or an empty map. */ + public ImmutableMap getMetadata(BlobId blobId) throws StorageException { + Blob blob = storage().get(blobId); + return blob == null ? ImmutableMap.of() : ImmutableMap.copyOf(blob.getMetadata()); + } + + /** + * Returns the {@link BlobInfo} of the given GCS file. + * + *

Note that a {@link Blob} is returned, but on the {@link BlobInfo} part of it is usable. + */ + public BlobInfo getBlobInfo(BlobId blobId) throws StorageException { + return storage().get(blobId); + } + + /** Determines most appropriate {@link BlobInfo} based on filename extension. */ + private static BlobInfo createBlobInfo(BlobId blobId) { + BlobInfo.Builder builder = BlobInfo.newBuilder(blobId).setCacheControl("no-cache"); + MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(blobId.getName()))); if (mediaType != null) { - builder = builder.mimeType(mediaType.type()); + builder = builder.setContentType(mediaType.toString()); } return builder.build(); } + + // These two methods are needed to check whether serialization is done correctly in tests. + @Override + public boolean equals(Object obj) { + return obj instanceof GcsUtils && ((GcsUtils) obj).storageOptions.equals(storageOptions); + } + + @Override + public int hashCode() { + return storageOptions.hashCode(); + } } diff --git a/core/src/main/java/google/registry/module/backend/BackendComponent.java b/core/src/main/java/google/registry/module/backend/BackendComponent.java index 17f0b9153..3ac38490a 100644 --- a/core/src/main/java/google/registry/module/backend/BackendComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendComponent.java @@ -26,7 +26,6 @@ import google.registry.export.datastore.DatastoreAdminModule; import google.registry.export.sheet.SheetsServiceModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; -import google.registry.gcs.GcsServiceModule; import google.registry.groups.DirectoryModule; import google.registry.groups.GroupsModule; import google.registry.groups.GroupssettingsModule; @@ -64,7 +63,6 @@ import javax.inject.Singleton; DirectoryModule.class, DummyKeyringModule.class, DriveModule.class, - GcsServiceModule.class, GroupsModule.class, GroupssettingsModule.class, JSchModule.class, diff --git a/core/src/main/java/google/registry/module/tools/ToolsComponent.java b/core/src/main/java/google/registry/module/tools/ToolsComponent.java index bfa012ef0..f1a6f806c 100644 --- a/core/src/main/java/google/registry/module/tools/ToolsComponent.java +++ b/core/src/main/java/google/registry/module/tools/ToolsComponent.java @@ -22,7 +22,6 @@ import google.registry.config.RegistryConfig.ConfigModule; import google.registry.export.DriveModule; import google.registry.flows.ServerTridProviderModule; import google.registry.flows.custom.CustomLogicFactoryModule; -import google.registry.gcs.GcsServiceModule; import google.registry.groups.DirectoryModule; import google.registry.groups.GroupsModule; import google.registry.groups.GroupssettingsModule; @@ -54,7 +53,6 @@ import javax.inject.Singleton; DirectoryModule.class, DummyKeyringModule.class, DriveModule.class, - GcsServiceModule.class, GroupsModule.class, GroupssettingsModule.class, Jackson2Module.class, diff --git a/core/src/main/java/google/registry/rde/BrdaCopyAction.java b/core/src/main/java/google/registry/rde/BrdaCopyAction.java index 75a57cd15..fbe6777c8 100644 --- a/core/src/main/java/google/registry/rde/BrdaCopyAction.java +++ b/core/src/main/java/google/registry/rde/BrdaCopyAction.java @@ -17,7 +17,7 @@ package google.registry.rde; import static google.registry.model.rde.RdeMode.THIN; import static google.registry.request.Action.Method.POST; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.flogger.FluentLogger; import com.google.common.io.ByteStreams; import google.registry.config.RegistryConfig.Config; @@ -85,10 +85,10 @@ public final class BrdaCopyAction implements Runnable { private void copyAsRyde() throws IOException { String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0); - GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde"); - GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length"); - GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde"); - GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig"); + BlobId xmlFilename = BlobId.of(stagingBucket, prefix + ".xml.ghostryde"); + BlobId xmlLengthFilename = BlobId.of(stagingBucket, prefix + ".xml.length"); + BlobId rydeFile = BlobId.of(brdaBucket, prefix + ".ryde"); + BlobId sigFile = BlobId.of(brdaBucket, prefix + ".sig"); long xmlLength = readXmlLength(xmlLengthFilename); @@ -107,7 +107,7 @@ public final class BrdaCopyAction implements Runnable { } /** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */ - private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException { + private long readXmlLength(BlobId xmlLengthFilename) throws IOException { try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) { return Ghostryde.readLength(input); } diff --git a/core/src/main/java/google/registry/rde/RdeReportAction.java b/core/src/main/java/google/registry/rde/RdeReportAction.java index 72112a71b..2897d341f 100644 --- a/core/src/main/java/google/registry/rde/RdeReportAction.java +++ b/core/src/main/java/google/registry/rde/RdeReportAction.java @@ -23,7 +23,7 @@ import static google.registry.persistence.transaction.TransactionManagerUtil.tra import static google.registry.request.Action.Method.POST; import static google.registry.util.DateTimeUtils.isBeforeOrAt; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.flogger.FluentLogger; import com.google.common.io.ByteStreams; import google.registry.config.RegistryConfig.Config; @@ -97,7 +97,7 @@ public final class RdeReportAction implements Runnable, EscrowTask { .orElseThrow( () -> new IllegalStateException("RdeRevision was not set on generated deposit")); String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision); - GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde"); + BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde"); verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename); reporter.send(readReportFromGcs(reportFilename)); response.setContentType(PLAIN_TEXT_UTF_8); @@ -106,7 +106,7 @@ public final class RdeReportAction implements Runnable, EscrowTask { } /** Reads and decrypts the XML file from cloud storage. */ - private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException { + private byte[] readReportFromGcs(BlobId reportFilename) throws IOException { try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename); InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey)) { return ByteStreams.toByteArray(ghostrydeDecoder); diff --git a/core/src/main/java/google/registry/rde/RdeStagingAction.java b/core/src/main/java/google/registry/rde/RdeStagingAction.java index c3ac547a7..af75770dd 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingAction.java +++ b/core/src/main/java/google/registry/rde/RdeStagingAction.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Multimaps; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; +import google.registry.gcs.GcsUtils; import google.registry.mapreduce.MapreduceRunner; import google.registry.mapreduce.inputs.EppResourceInputs; import google.registry.mapreduce.inputs.NullInput; @@ -206,6 +207,7 @@ public final class RdeStagingAction implements Runnable { @Inject PendingDepositChecker pendingDepositChecker; @Inject RdeStagingReducer.Factory reducerFactory; @Inject Response response; + @Inject GcsUtils gcsUtils; @Inject MapreduceRunner mrRunner; @Inject @Config("transactionCooldown") Duration transactionCooldown; @Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual; @@ -234,7 +236,7 @@ public final class RdeStagingAction implements Runnable { } ValidationMode validationMode = lenient ? LENIENT : STRICT; RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings); - RdeStagingReducer reducer = reducerFactory.create(validationMode); + RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils); mrRunner .setJobName("Stage escrow deposits for all TLDs") diff --git a/core/src/main/java/google/registry/rde/RdeStagingReducer.java b/core/src/main/java/google/registry/rde/RdeStagingReducer.java index a6c0db649..f62c9ac46 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingReducer.java +++ b/core/src/main/java/google/registry/rde/RdeStagingReducer.java @@ -16,7 +16,6 @@ package google.registry.rde; import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime; @@ -24,10 +23,9 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.RetryParams; import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; @@ -69,27 +67,27 @@ public final class RdeStagingReducer extends Reducer new IllegalStateException("RdeRevision was not set on generated deposit")); final String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision); - final GcsFilename xmlFilename = new GcsFilename(bucket, name + ".xml.ghostryde"); - final GcsFilename xmlLengthFilename = new GcsFilename(bucket, name + ".xml.length"); - GcsFilename reportFilename = new GcsFilename(bucket, name + "-report.xml.ghostryde"); + final BlobId xmlFilename = BlobId.of(bucket, name + ".xml.ghostryde"); + final BlobId xmlLengthFilename = BlobId.of(bucket, name + ".xml.length"); + BlobId reportFilename = BlobId.of(bucket, name + "-report.xml.ghostryde"); verifyFileExists(xmlFilename); verifyFileExists(xmlLengthFilename); verifyFileExists(reportFilename); @@ -187,29 +187,30 @@ public final class RdeUploadAction implements Runnable, EscrowTask { } /** - * Performs a blocking upload of a cloud storage XML file to escrow provider, converting - * it to the RyDE format along the way by applying tar+compress+encrypt+sign, and saving the - * created RyDE file on GCS for future reference. + * Performs a blocking upload of a cloud storage XML file to escrow provider, converting it to the + * RyDE format along the way by applying tar+compress+encrypt+sign, and saving the created RyDE + * file on GCS for future reference. * *

This is done by layering a bunch of {@link java.io.FilterOutputStream FilterOutputStreams} - * on top of each other in reverse order that turn XML bytes into a RyDE file while - * simultaneously uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to - * blocking-copy bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline. + * on top of each other in reverse order that turn XML bytes into a RyDE file while simultaneously + * uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to blocking-copy + * bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline. * *

In pseudo-shell, the whole process looks like the following: * - * 

   {@code
-   *   gcs read $xmlFile \                                   # Get GhostRyDE from cloud storage.
-   *     | decrypt | decompress \                            # Convert it to XML.
-   *     | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
-   *     | tee gs://bucket/$rydeFilename.ryde \              # Save a copy of the RyDE file to GCS.
-   *     | sftp put $dstUrl/$rydeFilename.ryde \             # Upload to SFTP server.
-   *    && sftp put $dstUrl/$rydeFilename.sig  gs://bucket/$rydeFilename.sig      # Save a copy of signature to GCS.
-   *   }
+ * 
{@code
+   * gcs read $xmlFile \                                   # Get GhostRyDE from cloud storage.
+   *   | decrypt | decompress \                            # Convert it to XML.
+   *   | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
+   *   | tee gs://bucket/$rydeFilename.ryde \              # Save a copy of the RyDE file to GCS.
+   *   | sftp put $dstUrl/$rydeFilename.ryde \             # Upload to SFTP server.
+   *  && sftp put $dstUrl/$rydeFilename.sig  gs://bucket/$rydeFilename.sig      # Save a copy of signature to GCS.
+   *
+   * }
*/ @VisibleForTesting - protected void upload(GcsFilename xmlFile, long xmlLength, DateTime watermark, String name) + protected void upload(BlobId xmlFile, long xmlLength, DateTime watermark, String name) throws Exception { logger.atInfo().log("Uploading XML file '%s' to remote path '%s'.", xmlFile, uploadUrl); try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile); @@ -218,7 +219,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask { JSchSftpChannel ftpChan = session.openSftpChannel()) { ByteArrayOutputStream sigOut = new ByteArrayOutputStream(); String rydeFilename = name + ".ryde"; - GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename); + BlobId rydeGcsFilename = BlobId.of(bucket, rydeFilename); try (OutputStream ftpOutput = ftpChan.get().put(rydeFilename, sftpProgressMonitor, OVERWRITE); OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename); @@ -234,7 +235,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask { } String sigFilename = name + ".sig"; byte[] signature = sigOut.toByteArray(); - gcsUtils.createFromBytes(new GcsFilename(bucket, sigFilename), signature); + gcsUtils.createFromBytes(BlobId.of(bucket, sigFilename), signature); ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename); logger.atInfo().log("uploaded %,d bytes: %s", signature.length, sigFilename); } @@ -242,13 +243,13 @@ public final class RdeUploadAction implements Runnable, EscrowTask { } /** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */ - private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException { + private long readXmlLength(BlobId xmlLengthFilename) throws IOException { try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) { return Ghostryde.readLength(input); } } - private void verifyFileExists(GcsFilename filename) { + private void verifyFileExists(BlobId filename) { verify(gcsUtils.existsAndNotEmpty(filename), "Missing file: %s", filename); } } diff --git a/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java b/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java index f74fa9268..324a9315e 100644 --- a/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java +++ b/core/src/main/java/google/registry/reporting/billing/BillingEmailUtils.java @@ -17,7 +17,7 @@ package google.registry.reporting.billing; import static com.google.common.base.Throwables.getRootCause; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.io.CharStreams; import com.google.common.net.MediaType; @@ -72,8 +72,7 @@ public class BillingEmailUtils { void emailOverallInvoice() { try { String invoiceFile = String.format("%s-%s.csv", invoiceFilePrefix, yearMonth); - GcsFilename invoiceFilename = - new GcsFilename(billingBucket, invoiceDirectoryPrefix + invoiceFile); + BlobId invoiceFilename = BlobId.of(billingBucket, invoiceDirectoryPrefix + invoiceFile); try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) { emailService.sendEmail( EmailMessage.newBuilder() diff --git a/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java b/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java index 8da9be513..b885fdf17 100644 --- a/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java +++ b/core/src/main/java/google/registry/reporting/billing/CopyDetailReportsAction.java @@ -19,7 +19,7 @@ import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -120,7 +120,7 @@ public final class CopyDetailReportsAction implements Runnable { () -> { try (InputStream input = gcsUtils.openInputStream( - new GcsFilename(billingBucket, invoiceDirectoryPrefix + detailReportName))) { + BlobId.of(billingBucket, invoiceDirectoryPrefix + detailReportName))) { driveConnection.createOrUpdateFile( detailReportName, MediaType.CSV_UTF_8, diff --git a/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java b/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java index ef1213fab..557d9b616 100644 --- a/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java +++ b/core/src/main/java/google/registry/reporting/icann/IcannReportingStager.java @@ -21,7 +21,7 @@ import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE import static java.nio.charset.StandardCharsets.UTF_8; import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.base.Ascii; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableCollection; @@ -260,7 +260,7 @@ public class IcannReportingStager { Ascii.toLowerCase(reportType.toString()), DateTimeFormat.forPattern("yyyyMM").print(yearMonth)); String reportBucketname = String.format("%s/%s", reportingBucket, subdir); - final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename); + final BlobId gcsFilename = BlobId.of(reportBucketname, reportFilename); gcsUtils.createFromBytes(gcsFilename, reportBytes); logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename); return reportFilename; @@ -269,7 +269,7 @@ public class IcannReportingStager { /** Creates and stores a manifest file on GCS, indicating which reports were generated. */ void createAndUploadManifest(String subdir, ImmutableList filenames) throws IOException { String reportBucketname = String.format("%s/%s", reportingBucket, subdir); - final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME); + final BlobId gcsFilename = BlobId.of(reportBucketname, MANIFEST_FILE_NAME); StringBuilder manifestString = new StringBuilder(); filenames.forEach((filename) -> manifestString.append(filename).append("\n")); gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8)); diff --git a/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java b/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java index 6860e19b1..a6b1c2dc7 100644 --- a/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java +++ b/core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java @@ -20,7 +20,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; @@ -147,7 +147,7 @@ public final class IcannReportingUploadAction implements Runnable { cursorTimeMinusMonth.getYear(), cursorTimeMinusMonth.getMonthOfYear()); String reportBucketname = String.format("%s/%s", reportingBucket, reportSubdir); String filename = getFileName(cursorType, cursorTime, tldStr); - final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename); + final BlobId gcsFilename = BlobId.of(reportBucketname, filename); logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname); // Check that the report exists try { @@ -298,18 +298,18 @@ public final class IcannReportingUploadAction implements Runnable { emailService.sendEmail(EmailMessage.create(subject, body, recipient, sender)); } - private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException { + private byte[] readBytesFromGcs(BlobId reportFilename) throws IOException { try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) { return ByteStreams.toByteArray(gcsInput); } } - private void verifyFileExists(GcsFilename gcsFilename) { + private void verifyFileExists(BlobId gcsFilename) { checkArgument( gcsUtils.existsAndNotEmpty(gcsFilename), "Object %s in bucket %s not found", - gcsFilename.getObjectName(), - gcsFilename.getBucketName()); + gcsFilename.getName(), + gcsFilename.getBucket()); } } diff --git a/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java b/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java index d3adb241c..57d9dcd79 100644 --- a/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java +++ b/core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java @@ -17,7 +17,7 @@ package google.registry.reporting.spec11; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -62,7 +62,7 @@ public class Spec11RegistrarThreatMatchesParser { } /** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */ - public ImmutableSet getFromFile(GcsFilename spec11ReportFilename) + public ImmutableSet getFromFile(BlobId spec11ReportFilename) throws IOException { if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) { return ImmutableSet.of(); @@ -81,7 +81,7 @@ public class Spec11RegistrarThreatMatchesParser { public Optional getPreviousDateWithMatches(LocalDate date) { LocalDate yesterday = date.minusDays(1); - GcsFilename gcsFilename = getGcsFilename(yesterday); + BlobId gcsFilename = getGcsFilename(yesterday); if (gcsUtils.existsAndNotEmpty(gcsFilename)) { return Optional.of(yesterday); } @@ -98,8 +98,8 @@ public class Spec11RegistrarThreatMatchesParser { return Optional.empty(); } - private GcsFilename getGcsFilename(LocalDate localDate) { - return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate)); + private BlobId getGcsFilename(LocalDate localDate) { + return BlobId.of(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate)); } private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException { diff --git a/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java b/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java index 947b0846f..0fadac0bb 100644 --- a/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java +++ b/core/src/main/java/google/registry/tools/server/GenerateZoneFilesAction.java @@ -14,7 +14,6 @@ package google.registry.tools.server; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterators.filter; import static com.google.common.io.BaseEncoding.base16; @@ -25,11 +24,10 @@ import static google.registry.request.Action.Method.POST; import static java.nio.charset.StandardCharsets.UTF_8; import static org.joda.time.DateTimeZone.UTC; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.RetryParams; import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -98,13 +96,14 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA @Inject MapreduceRunner mrRunner; @Inject JsonActionRunner jsonActionRunner; @Inject @Config("zoneFilesBucket") String bucket; - @Inject @Config("gcsBufferSize") int gcsBufferSize; @Inject @Config("commitLogDatastoreRetention") Duration datastoreRetention; @Inject @Config("dnsDefaultATtl") Duration dnsDefaultATtl; @SuppressWarnings("DurationVariableWithUnits") // false-positive Error Prone check @Inject @Config("dnsDefaultNsTtl") Duration dnsDefaultNsTtl; @Inject @Config("dnsDefaultDsTtl") Duration dnsDefaultDsTtl; @Inject Clock clock; + @Inject GcsUtils gcsUtils; + @Inject GenerateZoneFilesAction() {} @Override @@ -140,7 +139,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA .runMapreduce( new GenerateBindFileMapper( tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl), - new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize), + new GenerateBindFileReducer(bucket, exportTime, gcsUtils), ImmutableList.of(new NullInput<>(), createEntityInput(DomainBase.class))) .getLinkToMapreduceConsole(); ImmutableList filenames = @@ -236,22 +235,19 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA private final String bucket; private final DateTime exportTime; - private final int gcsBufferSize; + private final GcsUtils gcsUtils; - GenerateBindFileReducer(String bucket, DateTime exportTime, int gcsBufferSize) { + GenerateBindFileReducer(String bucket, DateTime exportTime, GcsUtils gcsUtils) { this.bucket = bucket; this.exportTime = exportTime; - this.gcsBufferSize = gcsBufferSize; + this.gcsUtils = gcsUtils; } @Override public void reduce(String tld, ReducerInput stanzas) { String stanzaCounter = tld + " stanzas"; - GcsFilename filename = - new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime)); - GcsUtils cloudStorage = - new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize); - try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename); + BlobId filename = BlobId.of(bucket, String.format(FILENAME_FORMAT, tld, exportTime)); + try (OutputStream gcsOutput = gcsUtils.openOutputStream(filename); Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8); PrintWriter writer = new PrintWriter(osWriter)) { writer.printf(HEADER_FORMAT, tld); diff --git a/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java b/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java index d75b54fe2..60d6c3761 100644 --- a/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java +++ b/core/src/test/java/google/registry/backup/ExportCommitLogDiffActionTest.java @@ -24,18 +24,17 @@ import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.util.DateTimeUtils.START_OF_TIME; import static org.joda.time.DateTimeZone.UTC; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableMap; import com.googlecode.objectify.Key; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.ImmutableObject; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; import google.registry.testing.AppEngineExtension; -import google.registry.testing.GcsTestingUtils; import google.registry.testing.TestObject; import java.util.List; import org.joda.time.DateTime; @@ -53,8 +52,7 @@ public class ExportCommitLogDiffActionTest { .withOfyTestEntities(TestObject.class) .build(); - /** Local GCS service available for testing. */ - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final DateTime now = DateTime.now(UTC); private final DateTime oneMinuteAgo = now.minusMinutes(1); @@ -63,7 +61,7 @@ public class ExportCommitLogDiffActionTest { @BeforeEach void beforeEach() { - task.gcsService = gcsService; + task.gcsUtils = gcsUtils; task.gcsBucket = "gcs bucket"; task.batchSize = 5; } @@ -84,10 +82,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -95,8 +94,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "0"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported).containsExactly(upperCheckpoint); } @@ -139,10 +137,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -150,8 +149,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "4"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in time order, with matching mutations. CommitLogManifest manifest1 = createManifest(2, now.minusDays(1).minusMillis(1)); @@ -191,10 +189,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -202,8 +201,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "4"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in the order below, with matching mutations. CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo); @@ -246,10 +244,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -257,8 +256,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "6"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in the order below, with matching mutations. CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo); @@ -301,10 +299,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -312,8 +311,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "0"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); // We expect no manifests or mutations, only the upper checkpoint. assertThat(exported).containsExactly(upperCheckpoint); } @@ -359,11 +357,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)) - .isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, oneMinuteAgo.toString(), @@ -371,8 +369,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "6"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in time order, with matching mutations. CommitLogManifest manifest1 = createManifest(3, oneMinuteAgo.minusDays(2)); @@ -415,10 +412,11 @@ public class ExportCommitLogDiffActionTest { task.run(); - GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now); + BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now); assertWithMessage("GCS file not found: " + expectedFilename) - .that(gcsService.getMetadata(expectedFilename)).isNotNull(); - assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata()) + .that(gcsUtils.existsAndNotEmpty(expectedFilename)) + .isTrue(); + assertThat(gcsUtils.getMetadata(expectedFilename)) .containsExactly( LOWER_BOUND_CHECKPOINT, START_OF_TIME.toString(), @@ -426,8 +424,7 @@ public class ExportCommitLogDiffActionTest { now.toString(), NUM_TRANSACTIONS, "3"); - List exported = - deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename)); + List exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename)); assertThat(exported.get(0)).isEqualTo(upperCheckpoint); // We expect these manifests, in the order below, with matching mutations. CommitLogManifest manifest1 = createManifest(1, START_OF_TIME.plusMillis(1)); diff --git a/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java b/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java index 7d4dd5de0..ff9a67d69 100644 --- a/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java +++ b/core/src/test/java/google/registry/backup/GcsDiffFileListerTest.java @@ -19,29 +19,24 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT; import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX; -import static java.lang.reflect.Proxy.newProxyInstance; import static org.joda.time.DateTimeZone.UTC; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFileMetadata; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; -import com.google.appengine.tools.cloudstorage.ListItem; -import com.google.appengine.tools.cloudstorage.ListResult; -import com.google.common.collect.Iterators; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.flogger.LoggerConfig; import com.google.common.testing.TestLogHandler; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.testing.AppEngineExtension; import java.io.IOException; -import java.lang.reflect.InvocationHandler; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.Callable; import java.util.logging.LogRecord; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; @@ -55,7 +50,7 @@ public class GcsDiffFileListerTest { private final DateTime now = DateTime.now(UTC); private final GcsDiffFileLister diffLister = new GcsDiffFileLister(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final TestLogHandler logHandler = new TestLogHandler(); @RegisterExtension @@ -64,25 +59,18 @@ public class GcsDiffFileListerTest { @BeforeEach void beforeEach() throws Exception { - diffLister.gcsService = gcsService; + diffLister.gcsUtils = gcsUtils; diffLister.executor = newDirectExecutorService(); for (int i = 0; i < 5; i++) { - gcsService.createOrReplace( - new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(i)), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(i + 1).toString()) - .build(), - ByteBuffer.wrap(new byte[]{1, 2, 3})); + addGcsFile(i, i + 1); } LoggerConfig.getConfig(GcsDiffFileLister.class).addHandler(logHandler); } - private Iterable extractTimesFromDiffFiles(List diffFiles) { + private Iterable extractTimesFromDiffFiles(ImmutableList diffFiles) { return transform( diffFiles, - metadata -> - DateTime.parse( - metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()))); + blobInfo -> DateTime.parse(blobInfo.getName().substring(DIFF_FILE_PREFIX.length()))); } private Iterable listDiffFiles(DateTime fromTime, DateTime toTime) { @@ -90,12 +78,12 @@ public class GcsDiffFileListerTest { } private void addGcsFile(int fileAge, int prevAge) throws IOException { - gcsService.createOrReplace( - new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString()) - .build(), - ByteBuffer.wrap(new byte[]{1, 2, 3})); + BlobInfo blobInfo = + BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge))) + .setMetadata( + ImmutableMap.of(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString())) + .build(); + gcsUtils.createFromBytes(blobInfo, new byte[] {1, 2, 3}); } private void assertLogContains(String message) { @@ -114,38 +102,11 @@ public class GcsDiffFileListerTest { } @Test - void testList_patchesHoles() { - // Fake out the GCS list() method to return only the first and last file. - // We can't use Mockito.spy() because GcsService's impl is final. - diffLister.gcsService = (GcsService) newProxyInstance( - GcsService.class.getClassLoader(), - new Class[] {GcsService.class}, - new InvocationHandler() { - @Override - public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - if (method.getName().equals("list")) { - // ListResult is an incredibly annoying thing to construct. It needs to be fed from a - // Callable that returns Iterators, each representing a batch of results. - return new ListResult(new Callable>() { - boolean called = false; - - @Override - public Iterator call() { - try { - return called ? null : Iterators.forArray( - new ListItem.Builder() - .setName(DIFF_FILE_PREFIX + now) - .build(), - new ListItem.Builder() - .setName(DIFF_FILE_PREFIX + now.minusMinutes(4)) - .build()); - } finally { - called = true; - } - }}); - } - return method.invoke(gcsService, args); - }}); + void testList_patchesHoles() throws Exception { + GcsUtils mockGcsUtils = mock(GcsUtils.class); + diffLister.gcsUtils = spy(gcsUtils); + when(mockGcsUtils.listFolderObjects(anyString(), anyString())) + .thenReturn(ImmutableList.of(now.toString(), now.minusMinutes(4).toString())); DateTime fromTime = now.minusMinutes(4).minusSeconds(1); // Request all files with checkpoint > fromTime. assertThat(listDiffFiles(fromTime, null)) diff --git a/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java b/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java index 693f015ac..40c7a2db1 100644 --- a/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java +++ b/core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java @@ -37,14 +37,14 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; import com.google.common.truth.Truth8; import com.googlecode.objectify.Key; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.common.DatabaseMigrationStateSchedule; import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; import google.registry.model.contact.ContactResource; @@ -110,7 +110,7 @@ public class ReplayCommitLogsToSqlActionTest { .build(); /** Local GCS service. */ - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final ReplayCommitLogsToSqlAction action = new ReplayCommitLogsToSqlAction(); private final FakeResponse response = new FakeResponse(); @@ -123,13 +123,13 @@ public class ReplayCommitLogsToSqlActionTest { @BeforeEach void beforeEach() { - action.gcsService = gcsService; + action.gcsUtils = gcsUtils; action.response = response; action.requestStatusChecker = requestStatusChecker; action.clock = fakeClock; action.gcsBucket = "gcs bucket"; action.diffLister = new GcsDiffFileLister(); - action.diffLister.gcsService = gcsService; + action.diffLister.gcsUtils = gcsUtils; action.diffLister.executor = newDirectExecutorService(); ofyTm() .transact( @@ -165,9 +165,9 @@ public class ReplayCommitLogsToSqlActionTest { CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2)); Key manifest2Key = CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1)); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -182,7 +182,7 @@ public class ReplayCommitLogsToSqlActionTest { CommitLogMutation.create(manifest1bKey, TestObject.create("c")), CommitLogMutation.create(manifest1bKey, TestObject.create("d"))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -200,8 +200,8 @@ public class ReplayCommitLogsToSqlActionTest { void testReplay_noManifests() throws Exception { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep"))); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); - saveDiffFile(gcsService, createCheckpoint(now.minusMillis(2))); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); + saveDiffFile(gcsUtils, createCheckpoint(now.minusMillis(2))); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1))); runAndAssertSuccess(now.minusMillis(1)); assertExpectedIds("previous to keep"); @@ -214,10 +214,10 @@ public class ReplayCommitLogsToSqlActionTest { jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep"))); Key bucketKey = getBucketKey(1); Key manifestKey = CommitLogManifest.createKey(bucketKey, now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a")), @@ -237,10 +237,10 @@ public class ReplayCommitLogsToSqlActionTest { jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep"))); Key bucketKey = getBucketKey(1); Key manifestKey = CommitLogManifest.createKey(bucketKey, now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a")), @@ -258,10 +258,10 @@ public class ReplayCommitLogsToSqlActionTest { jpaTm().insertWithoutBackup(TestObject.create("previous to keep")); jpaTm().insertWithoutBackup(TestObject.create("previous to delete")); }); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -276,10 +276,10 @@ public class ReplayCommitLogsToSqlActionTest { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> jpaTm().put(TestObject.create("existing", "a"))); Key manifestKey = CommitLogManifest.createKey(getBucketKey(1), now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMillis(1)), CommitLogManifest.create(getBucketKey(1), now, null), CommitLogMutation.create(manifestKey, TestObject.create("existing", "b"))); @@ -294,10 +294,10 @@ public class ReplayCommitLogsToSqlActionTest { void testReplay_deleteMissingEntity() throws Exception { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> jpaTm().put(TestObject.create("previous to keep", "a"))); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1)); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMillis(1)), CommitLogManifest.create( getBucketKey(1), @@ -332,7 +332,7 @@ public class ReplayCommitLogsToSqlActionTest { TransactionManagerFactory.setJpaTm(() -> spy); // Save in the commit logs the domain and contact (in that order) and the token deletion saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(toDelete))), @@ -375,7 +375,7 @@ public class ReplayCommitLogsToSqlActionTest { TransactionManagerFactory.setJpaTm(() -> spy); // Save two commits -- the deletion, then the new version of the contact saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1).plusMillis(1)), CommitLogManifest.create( getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(contact))), @@ -415,7 +415,7 @@ public class ReplayCommitLogsToSqlActionTest { () -> { try { saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), now.minusMinutes(1), ImmutableSet.of()), @@ -444,7 +444,7 @@ public class ReplayCommitLogsToSqlActionTest { // Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant) Key claimsListKey = Key.create(ClaimsList.class, 1L); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -491,7 +491,7 @@ public class ReplayCommitLogsToSqlActionTest { Key manifestKey = CommitLogManifest.createKey(bucketKey, now); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a"))); @@ -504,7 +504,7 @@ public class ReplayCommitLogsToSqlActionTest { DateTime now = fakeClock.nowUtc(); jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1))); saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), diff --git a/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java b/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java index 9fd9c3522..2b72b060c 100644 --- a/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java +++ b/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java @@ -14,7 +14,6 @@ package google.registry.backup; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Maps.toMap; import static com.google.common.truth.Truth.assertThat; @@ -28,15 +27,16 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy; import static org.joda.time.DateTimeZone.UTC; import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.io.Resources; import com.google.common.primitives.Longs; import com.googlecode.objectify.Key; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.ImmutableObject; import google.registry.model.domain.DomainBase; import google.registry.model.ofy.CommitLogBucket; @@ -51,7 +51,6 @@ import google.registry.testing.TestObject; import google.registry.util.Retrier; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -68,7 +67,7 @@ public class RestoreCommitLogsActionTest { private final DateTime now = DateTime.now(UTC); private final RestoreCommitLogsAction action = new RestoreCommitLogsAction(); - private final GcsService gcsService = createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @RegisterExtension public final AppEngineExtension appEngine = @@ -79,7 +78,7 @@ public class RestoreCommitLogsActionTest { @BeforeEach void beforeEach() { - action.gcsService = gcsService; + action.gcsUtils = gcsUtils; action.dryRun = false; action.datastoreService = DatastoreServiceFactory.getDatastoreService(); action.fromTime = now.minusMillis(1); @@ -87,7 +86,7 @@ public class RestoreCommitLogsActionTest { action.defaultGcsBucket = GCS_BUCKET; action.gcsBucketOverride = Optional.empty(); action.diffLister = new GcsDiffFileLister(); - action.diffLister.gcsService = gcsService; + action.diffLister.gcsUtils = gcsUtils; action.diffLister.executor = newDirectExecutorService(); } @@ -109,10 +108,10 @@ public class RestoreCommitLogsActionTest { CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2)); Key manifest2Key = CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1)); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(2)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2)); Iterable file1CommitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now.minusMinutes(1)), CommitLogManifest.create( getBucketKey(1), @@ -128,7 +127,7 @@ public class RestoreCommitLogsActionTest { CommitLogMutation.create(manifest1bKey, TestObject.create("d"))); Iterable file2CommitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -149,8 +148,8 @@ public class RestoreCommitLogsActionTest { @Test void testRestore_noManifests() throws Exception { auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); - Iterable commitLogs = saveDiffFile(gcsService, createCheckpoint(now)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); + Iterable commitLogs = saveDiffFile(gcsUtils, createCheckpoint(now)); action.run(); auditedOfy().clearSessionCache(); assertExpectedIds("previous to keep"); @@ -164,10 +163,10 @@ public class RestoreCommitLogsActionTest { auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now(); Key bucketKey = getBucketKey(1); Key manifestKey = CommitLogManifest.createKey(bucketKey, now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create(bucketKey, now, null), CommitLogMutation.create(manifestKey, TestObject.create("a")), @@ -186,10 +185,10 @@ public class RestoreCommitLogsActionTest { .saveWithoutBackup() .entities(TestObject.create("previous to keep"), TestObject.create("previous to delete")) .now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -207,12 +206,10 @@ public class RestoreCommitLogsActionTest { @Test void testRestore_manifestWithNoMutationsOrDeletions() throws Exception { auditedOfy().saveWithoutBackup().entities(TestObject.create("previous to keep")).now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, - createCheckpoint(now), - CommitLogManifest.create(getBucketKey(1), now, null)); + gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null)); action.run(); auditedOfy().clearSessionCache(); assertExpectedIds("previous to keep"); @@ -225,10 +222,10 @@ public class RestoreCommitLogsActionTest { void testRestore_mutateExistingEntity() throws Exception { auditedOfy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now(); Key manifestKey = CommitLogManifest.createKey(getBucketKey(1), now); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null), CommitLogMutation.create(manifestKey, TestObject.create("existing", "b"))); @@ -245,10 +242,10 @@ public class RestoreCommitLogsActionTest { @Test void testRestore_deleteMissingEntity() throws Exception { auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now(); - saveDiffFileNotToRestore(gcsService, now.minusMinutes(1)); + saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1)); Iterable commitLogs = saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create( getBucketKey(1), @@ -270,7 +267,7 @@ public class RestoreCommitLogsActionTest { // imported, in particular, the domain's 'registrant' key can be used by Objectify to load the // contact. saveDiffFile( - gcsService, + gcsUtils, Resources.toByteArray(Resources.getResource("google/registry/backup/commitlog.data")), now); action.run(); @@ -286,18 +283,18 @@ public class RestoreCommitLogsActionTest { return CommitLogCheckpoint.create(now, toMap(getBucketIds(), x -> now)); } - static void saveDiffFile(GcsService gcsService, byte[] rawBytes, DateTime timestamp) + static void saveDiffFile(GcsUtils gcsUtils, byte[] rawBytes, DateTime timestamp) throws IOException { - gcsService.createOrReplace( - new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString()) - .build(), - ByteBuffer.wrap(rawBytes)); + BlobInfo blobInfo = + BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp)) + .setMetadata( + ImmutableMap.of(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString())) + .build(); + gcsUtils.createFromBytes(blobInfo, rawBytes); } static Iterable saveDiffFile( - GcsService gcsService, CommitLogCheckpoint checkpoint, ImmutableObject... entities) + GcsUtils gcsUtils, CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException { DateTime now = checkpoint.getCheckpointTime(); List allEntities = Lists.asList(checkpoint, entities); @@ -305,13 +302,13 @@ public class RestoreCommitLogsActionTest { for (ImmutableObject entity : allEntities) { serializeEntity(entity, output); } - saveDiffFile(gcsService, output.toByteArray(), now); + saveDiffFile(gcsUtils, output.toByteArray(), now); return allEntities; } - static void saveDiffFileNotToRestore(GcsService gcsService, DateTime now) throws Exception { + static void saveDiffFileNotToRestore(GcsUtils gcsUtils, DateTime now) throws Exception { saveDiffFile( - gcsService, + gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null), CommitLogMutation.create( diff --git a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java index 431291ee1..2af29c6c9 100644 --- a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java +++ b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java @@ -14,14 +14,12 @@ package google.registry.export; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.truth.Truth.assertThat; import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_FILENAME; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistDeletedDomain; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.eq; @@ -29,12 +27,13 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.ListOptions; -import com.google.appengine.tools.cloudstorage.ListResult; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.StorageException; +import com.google.common.collect.ImmutableList; import com.google.common.net.MediaType; import google.registry.export.ExportDomainListsAction.ExportDomainListsReducer; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.ofy.Ofy; import google.registry.model.registry.Registry; import google.registry.model.registry.Registry.TldType; @@ -46,7 +45,6 @@ import google.registry.testing.InjectExtension; import google.registry.testing.TestOfyAndSql; import google.registry.testing.TestOfyOnly; import google.registry.testing.mapreduce.MapreduceTestCase; -import java.io.FileNotFoundException; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; @@ -57,7 +55,7 @@ import org.mockito.ArgumentCaptor; @DualDatabaseTest class ExportDomainListsActionTest extends MapreduceTestCase { - private GcsService gcsService; + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private DriveConnection driveConnection = mock(DriveConnection.class); private ArgumentCaptor bytesExportedToDrive = ArgumentCaptor.forClass(byte[].class); private final FakeResponse response = new FakeResponse(); @@ -81,10 +79,9 @@ class ExportDomainListsActionTest extends MapreduceTestCase readGcsFile(gcsService, nonexistentFile)); - ListResult ls = gcsService.list("outputbucket", ListOptions.DEFAULT); - assertThat(ls.next().getName()).isEqualTo("tld.txt"); - // Make sure that no other files were written out. - assertThat(ls.hasNext()).isFalse(); + BlobId nonexistentFile = BlobId.of("outputbucket", "testtld.txt"); + assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile)); + ImmutableList ls = gcsUtils.listFolderObjects("outputbucket", ""); + assertThat(ls).containsExactly("tld.txt"); verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld"); verifyNoMoreInteractions(driveConnection); } @@ -160,14 +155,14 @@ class ExportDomainListsActionTest extends MapreduceTestCase metadata = ImmutableMap.of("key1", "val1", "Key2", "val2"); + private final byte[] bytes = new byte[] {'a', 'b', 'c'}; + + @BeforeEach + void beforeEach() {} + + @Test + void testSerialization_testStorage() throws Exception { + assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils); + } + + @Test + void testSerialization_realStorage() throws Exception { + gcsUtils = new GcsUtils(StorageOptions.getDefaultInstance()); + assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils); + } + + @Test + void testStreams() throws Exception { + try (OutputStream os = gcsUtils.openOutputStream(blobId, metadata)) { + os.write(bytes); + os.flush(); + } + assertThat(ByteStreams.toByteArray(gcsUtils.openInputStream(blobId))).isEqualTo(bytes); + assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).containsExactlyEntriesIn(metadata); + } + + @Test + void testCreateListReadDelete() throws Exception { + gcsUtils.createFromBytes(BlobInfo.newBuilder(blobId).setMetadata(metadata).build(), bytes); + assertThat(gcsUtils.existsAndNotEmpty(blobId)).isTrue(); + assertThat(gcsUtils.listFolderObjects(bucket, "")).containsExactly("my-file"); + assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).isEqualTo(metadata); + assertThat(gcsUtils.readBytesFrom(blobId)).isEqualTo(bytes); + gcsUtils.delete(blobId); + assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse(); + assertThat(gcsUtils.listFolderObjects(bucket, "")).isEmpty(); + } + + @Test + void testSetContentType() { + blobId = BlobId.of(bucket, "something.json"); + gcsUtils.createFromBytes(blobId, bytes); + assertThat(gcsUtils.getBlobInfo(blobId).getContentType()) + .isEqualTo(MediaType.JSON_UTF_8.toString()); + } + + @Test + void testList() throws Exception { + ImmutableList blobIds = + ImmutableList.of( + BlobId.of(bucket, "a/b/xyz.txt"), + BlobId.of(bucket, "a/cde.exe"), + BlobId.of(bucket, "fgh.jpg"), + BlobId.of(bucket, "c/ijk.mp4")); + + for (BlobId blobId : blobIds) { + gcsUtils.createFromBytes(blobId, bytes); + } + assertThat(gcsUtils.listFolderObjects(bucket, "")) + .containsExactly("a/b/xyz.txt", "a/cde.exe", "fgh.jpg", "c/ijk.mp4"); + assertThat(gcsUtils.listFolderObjects(bucket, "a/")).containsExactly("b/xyz.txt", "cde.exe"); + } + + @Test + void testEmptyFile() { + gcsUtils.createFromBytes(blobId, new byte[] {}); + assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse(); + } + + private static byte[] serialize(Object object) throws IOException { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ObjectOutputStream oos = new ObjectOutputStream(baos); + oos.writeObject(object); + oos.flush(); + return baos.toByteArray(); + } + } + + private static Object deserialize(byte[] bytes) throws IOException, ClassNotFoundException { + try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) { + return ois.readObject(); + } + } +} diff --git a/core/src/test/java/google/registry/gcs/backport/FakeStorageRpc.java b/core/src/test/java/google/registry/gcs/backport/FakeStorageRpc.java new file mode 100644 index 000000000..74210e68e --- /dev/null +++ b/core/src/test/java/google/registry/gcs/backport/FakeStorageRpc.java @@ -0,0 +1,517 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package google.registry.gcs.backport; + +import com.google.api.client.util.DateTime; +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.ServiceAccount; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.Tuple; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.testing.StorageRpcTestBase; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.nio.file.FileAlreadyExistsException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.concurrent.NotThreadSafe; + +/** + * A bare-bones in-memory implementation of StorageRpc, meant for testing. + * + *

This class is not thread-safe. It's also (currently) limited in the following ways: + * + *

    + *
  • Supported + *
      + *
    • object create + *
    • object get + *
    • object delete + *
    • list the contents of a bucket + *
    • generations + *
    + *
  • Unsupported + *
      + *
    • bucket create + *
    • bucket get + *
    • bucket delete + *
    • list all buckets + *
    • file attributes + *
    • patch + *
    • continueRewrite + *
    • createBatch + *
    • checksums, etags + *
    • IAM operations + *
    • BucketLock operations + *
    • HMAC key operations + *
    + *
+ */ +@NotThreadSafe +class FakeStorageRpc extends StorageRpcTestBase { + + private static final SimpleDateFormat RFC_3339_FORMATTER = + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX"); + + // fullname -> metadata + Map metadata = new HashMap<>(); + // fullname -> contents + Map contents = new HashMap<>(); + // fullname -> future contents that will be visible on close. + Map futureContents = new HashMap<>(); + + private final boolean throwIfOption; + + /** @param throwIfOption if true, we throw when given any option */ + public FakeStorageRpc(boolean throwIfOption) { + this.throwIfOption = throwIfOption; + } + + // remove all files + void reset() { + metadata = new HashMap<>(); + contents = new HashMap<>(); + } + + @Override + public StorageObject create(StorageObject object, InputStream content, Map options) + throws StorageException { + potentiallyThrow(options); + String key = fullname(object); + object.setUpdated(now()); + metadata.put(key, object); + try { + contents.put(key, com.google.common.io.ByteStreams.toByteArray(content)); + } catch (IOException e) { + throw new StorageException(e); + } + // TODO: crc, etc + return object; + } + + @Override + public Tuple> list(String bucket, Map options) + throws StorageException { + String delimiter = null; + String preprefix = ""; + String pageToken = null; + long maxResults = Long.MAX_VALUE; + for (Map.Entry e : options.entrySet()) { + switch (e.getKey()) { + case PAGE_TOKEN: + pageToken = (String) e.getValue(); + break; + case PREFIX: + preprefix = (String) e.getValue(); + if (preprefix.startsWith("/")) { + preprefix = preprefix.substring(1); + } + break; + case DELIMITER: + delimiter = (String) e.getValue(); + break; + case FIELDS: + // ignore and return all the fields + break; + case MAX_RESULTS: + maxResults = (Long) e.getValue(); + break; + case USER_PROJECT: + // prevent unsupported operation + break; + default: + throw new UnsupportedOperationException("Unknown option: " + e.getKey()); + } + } + final String prefix = preprefix; + + List values = new ArrayList<>(); + Map folders = new HashMap<>(); + for (StorageObject so : metadata.values()) { + if (!so.getBucket().equals(bucket) || !so.getName().startsWith(prefix)) { + continue; + } + if (processedAsFolder(so, delimiter, prefix, folders)) { + continue; + } + so.setSize(size(so)); + values.add(so); + } + values.addAll(folders.values()); + + // truncate to max allowed length + if (values.size() > maxResults) { + List newValues = new ArrayList<>(); + for (int i = 0; i < maxResults; i++) { + newValues.add(values.get(i)); + } + values = newValues; + } + + // null cursor to indicate there is no more data (empty string would cause us to be called + // again). + // The type cast seems to be necessary to help Java's typesystem remember that collections are + // iterable. + return Tuple.of(pageToken, (Iterable) values); + } + + /** Returns the requested bucket or {@code null} if not found. */ + @Override + public Bucket get(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + /** Returns the requested storage object or {@code null} if not found. */ + @Override + public StorageObject get(StorageObject object, Map options) throws StorageException { + // we allow the "ID" option because we need to, but then we give a whole answer anyways + // because the caller won't mind the extra fields. + if (throwIfOption + && !options.isEmpty() + && options.size() > 1 + && options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) { + throw new UnsupportedOperationException(); + } + + String key = fullname(object); + if (metadata.containsKey(key)) { + StorageObject ret = metadata.get(key); + ret.setSize(size(ret)); + ret.setId(key); + + return ret; + } + return null; + } + + @Override + public Bucket patch(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public StorageObject patch(StorageObject storageObject, Map options) + throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public boolean delete(Bucket bucket, Map options) throws StorageException { + return false; + } + + @Override + public boolean delete(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + contents.remove(key); + return null != metadata.remove(key); + } + + @Override + public StorageObject compose( + Iterable sources, StorageObject target, Map targetOptions) + throws StorageException { + return null; + } + + @Override + public byte[] load(StorageObject storageObject, Map options) throws StorageException { + String key = fullname(storageObject); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + return contents.get(key); + } + + @Override + public Tuple read( + StorageObject from, Map options, long zposition, int zbytes) + throws StorageException { + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option op : options.keySet()) { + if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) { + generationMatch = (Long) options.get(op); + } else { + throw new UnsupportedOperationException("Unknown option: " + op); + } + } + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + checkGeneration(key, generationMatch); + long position = zposition; + int bytes = zbytes; + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + if ((int) position + bytes > full.length) { + bytes = full.length - (int) position; + } + if (bytes <= 0) { + // special case: you're trying to read past the end + return Tuple.of("etag-goes-here", new byte[0]); + } + byte[] ret = new byte[bytes]; + System.arraycopy(full, (int) position, ret, 0, bytes); + return Tuple.of("etag-goes-here", ret); + } + + @Override + public long read( + StorageObject from, Map options, long position, OutputStream outputStream) { + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option op : options.keySet()) { + if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) { + generationMatch = (Long) options.get(op); + } else { + throw new UnsupportedOperationException("Unknown option: " + op); + } + } + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + checkGeneration(key, generationMatch); + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + int bytes = (int) (full.length - position); + if (bytes <= 0) { + // special case: you're trying to read past the end + return 0; + } + try { + outputStream.write(full, (int) position, bytes); + } catch (IOException e) { + throw new StorageException(500, "Failed to write to file", e); + } + return bytes; + } + + @Override + public String open(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option option : options.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH) { + generationMatch = (Long) options.get(option); + } + } + checkGeneration(key, generationMatch); + metadata.put(key, object); + + return fullname(object); + } + + @Override + public String open(String signedURL) { + return null; + } + + @Override + public void write( + String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, int length, boolean last) + throws StorageException { + writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last); + } + + @Override + public StorageObject writeWithResponse( + String uploadId, + byte[] toWrite, + int toWriteOffset, + long destOffset, + int length, + boolean last) { + // this may have a lot more allocations than ideal, but it'll work. + byte[] bytes; + if (futureContents.containsKey(uploadId)) { + bytes = futureContents.get(uploadId); + if (bytes.length < length + destOffset) { + byte[] newBytes = new byte[(int) (length + destOffset)]; + System.arraycopy(bytes, 0, newBytes, (int) 0, bytes.length); + bytes = newBytes; + } + } else { + bytes = new byte[(int) (length + destOffset)]; + } + System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length); + // we want to mimic the GCS behavior that file contents are only visible on close. + StorageObject storageObject = null; + if (last) { + contents.put(uploadId, bytes); + futureContents.remove(uploadId); + if (metadata.containsKey(uploadId)) { + storageObject = metadata.get(uploadId); + storageObject.setUpdated(now()); + Long generation = storageObject.getGeneration(); + if (null == generation) { + generation = Long.valueOf(0); + } + storageObject.setGeneration(++generation); + metadata.put(uploadId, storageObject); + } + } else { + futureContents.put(uploadId, bytes); + } + return storageObject; + } + + @Override + public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException { + String sourceKey = fullname(rewriteRequest.source); + + // a little hackish, just good enough for the tests to work. + if (!contents.containsKey(sourceKey)) { + throw new StorageException(404, "File not found: " + sourceKey); + } + + // if non-null, then we check the file's at that generation. + Long generationMatch = null; + for (Option option : rewriteRequest.targetOptions.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH) { + generationMatch = (Long) rewriteRequest.targetOptions.get(option); + } + } + + String destKey = fullname(rewriteRequest.target); + + // if this is a new file, set generation to 1, else increment the existing generation + long generation = 1; + if (metadata.containsKey(destKey)) { + generation = metadata.get(destKey).getGeneration() + 1; + } + + checkGeneration(destKey, generationMatch); + + byte[] data = contents.get(sourceKey); + + rewriteRequest.target.setGeneration(generation); + rewriteRequest.target.setSize(BigInteger.valueOf(data.length)); + rewriteRequest.target.setUpdated(metadata.get(sourceKey).getUpdated()); + + metadata.put(destKey, rewriteRequest.target); + + contents.put(destKey, Arrays.copyOf(data, data.length)); + return new RewriteResponse( + rewriteRequest, + rewriteRequest.target, + data.length, + true, + "rewriteToken goes here", + data.length); + } + + private static DateTime now() { + return DateTime.parseRfc3339(RFC_3339_FORMATTER.format(new Date())); + } + + private String fullname(StorageObject so) { + return (so.getBucket() + "/" + so.getName()); + } + + private BigInteger size(StorageObject so) { + String key = fullname(so); + + if (contents.containsKey(key)) { + return BigInteger.valueOf(contents.get(key).length); + } + + return null; + } + + private void potentiallyThrow(Map options) throws UnsupportedOperationException { + if (throwIfOption && !options.isEmpty()) { + throw new UnsupportedOperationException(); + } + } + + /** + * Throw if we're asking for generation 0 and the file exists, or if the requested generation + * number doesn't match what is asked. + * + * @param key + * @param generationMatch + */ + private void checkGeneration(String key, Long generationMatch) { + if (null == generationMatch) { + return; + } + if (generationMatch == 0 && metadata.containsKey(key)) { + throw new StorageException(new FileAlreadyExistsException(key)); + } + if (generationMatch != 0) { + Long generation = metadata.get(key).getGeneration(); + if (!generationMatch.equals(generation)) { + throw new StorageException( + 404, "Generation mismatch. Requested " + generationMatch + " but got " + generation); + } + } + } + + // Returns true if this is a folder. Adds it to folders if it isn't already there. + private static boolean processedAsFolder( + StorageObject so, + String delimiter, + String prefix, /* inout */ + Map folders) { + if (delimiter == null) { + return false; + } + int nextSlash = so.getName().indexOf(delimiter, prefix.length()); + if (nextSlash < 0) { + return false; + } + String folderName = so.getName().substring(0, nextSlash + 1); + if (folders.containsKey(folderName)) { + return true; + } + StorageObject fakeFolder = new StorageObject(); + fakeFolder.setName(folderName); + fakeFolder.setBucket(so.getBucket()); + fakeFolder.setGeneration(so.getGeneration()); + fakeFolder.set("isDirectory", true); + fakeFolder.setSize(BigInteger.ZERO); + folders.put(folderName, fakeFolder); + return true; + } + + @Override + public ServiceAccount getServiceAccount(String projectId) { + return null; + } +} diff --git a/core/src/test/java/google/registry/gcs/backport/LocalStorageHelper.java b/core/src/test/java/google/registry/gcs/backport/LocalStorageHelper.java new file mode 100644 index 000000000..e4b51c348 --- /dev/null +++ b/core/src/test/java/google/registry/gcs/backport/LocalStorageHelper.java @@ -0,0 +1,106 @@ +/* + * Copyright 2016 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package google.registry.gcs.backport; + +import com.google.cloud.ServiceRpc; +import com.google.cloud.spi.ServiceRpcFactory; +import com.google.cloud.storage.StorageOptions; + +/** + * Utility to create an in-memory storage configuration for testing. Storage options can be obtained + * via the {@link #getOptions()} method. Returned options will point to FakeStorageRpc. + * + *

Note, the created in-memory storage configuration supports limited set of operations and is + * not thread-safe: + * + *

    + *
  • Supported operations + *
      + *
    • object create + *
    • object get + *
    • object delete + *
    • list the contents of a bucket + *
    + *
  • Unsupported operations + *
      + *
    • bucket create + *
    • bucket get + *
    • bucket delete + *
    • list all buckets + *
    • generations + *
    • file attributes + *
    • patch + *
    • continueRewrite + *
    • createBatch + *
    • checksums, etags + *
    • IAM operations + *
    + *
+ * + * {@link FakeStorageRpc#list(String, java.util.Map)} lists all the objects that have been created + * rather than the objects in the provided bucket. Since this class does not support creating, + * listing and deleting buckets, the parameter bucket here is not actually used and on serves as a + * placeholder. + */ +public final class LocalStorageHelper { + + // used for testing. Will throw if you pass it an option. + private static final FakeStorageRpc instance = new FakeStorageRpc(true); + + private LocalStorageHelper() {} + + /** + * Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, and resets it + * first so you start from a clean slate. That instance will throw if you pass it any option. + */ + public static StorageOptions getOptions() { + instance.reset(); + return StorageOptions.newBuilder() + .setProjectId("fake-project-for-testing") + .setServiceRpcFactory(new FakeStorageRpcFactory()) + .build(); + } + + /** + * Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance with the given + * option. + */ + public static StorageOptions customOptions(final boolean throwIfOptions) { + return StorageOptions.newBuilder() + .setProjectId("fake-project-for-testing") + .setServiceRpcFactory(new FakeStorageRpcFactory(new FakeStorageRpc(throwIfOptions))) + .build(); + } + + public static class FakeStorageRpcFactory implements ServiceRpcFactory { + + private final FakeStorageRpc instance; + + public FakeStorageRpcFactory() { + this(LocalStorageHelper.instance); + } + + public FakeStorageRpcFactory(FakeStorageRpc instance) { + this.instance = instance; + } + + @Override + public ServiceRpc create(StorageOptions storageOptions) { + return instance; + } + } +} diff --git a/core/src/test/java/google/registry/gcs/backport/README.md b/core/src/test/java/google/registry/gcs/backport/README.md new file mode 100644 index 000000000..caa714b7e --- /dev/null +++ b/core/src/test/java/google/registry/gcs/backport/README.md @@ -0,0 +1,9 @@ +The files here are directly lifted +from [googleapis/java-storage-nio](https://github.com/googleapis/java-storage-nio/tree/master/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing) +. They are needed because the `StorageOptions` returned from `LocalStorageHelper.getOptions()` +should be serializable for the `GcsUtils` class itself to be serializable in tests, but is not. The +bug is [fixed](https://github.com/googleapis/java-storage-nio/pull/606) upstream. However, the +current released package does not contain the fix yet. + +They are not put under `common/testing` because we do not want to introduce a dependency on the +testing configuration from core. \ No newline at end of file diff --git a/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java b/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java index 229b49827..426d90d81 100644 --- a/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java +++ b/core/src/test/java/google/registry/rde/BrdaCopyActionTest.java @@ -16,23 +16,20 @@ package google.registry.rde; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.SystemInfo.hasCommand; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assumptions.assumeTrue; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.io.ByteSource; import com.google.common.io.CharStreams; import com.google.common.io.Files; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.Keyring; import google.registry.testing.AppEngineExtension; import google.registry.testing.BouncyCastleProviderExtension; import google.registry.testing.FakeKeyringModule; -import google.registry.testing.GcsTestingUtils; import google.registry.testing.GpgSystemCommandExtension; import java.io.File; import java.io.IOException; @@ -52,14 +49,12 @@ public class BrdaCopyActionTest { private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml"); - private static final GcsFilename STAGE_FILE = - new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde"); - private static final GcsFilename STAGE_LENGTH_FILE = - new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.length"); - private static final GcsFilename RYDE_FILE = - new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.ryde"); - private static final GcsFilename SIG_FILE = - new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.sig"); + private static final BlobId STAGE_FILE = + BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde"); + private static final BlobId STAGE_LENGTH_FILE = + BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.length"); + private static final BlobId RYDE_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.ryde"); + private static final BlobId SIG_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.sig"); @RegisterExtension public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension(); @@ -89,8 +84,7 @@ public class BrdaCopyActionTest { } } - private final GcsService gcsService = GcsServiceFactory.createGcsService(); - private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final BrdaCopyAction action = new BrdaCopyAction(); @BeforeEach @@ -105,9 +99,8 @@ public class BrdaCopyActionTest { action.stagingDecryptionKey = decryptKey; byte[] xml = DEPOSIT_XML.read(); - GcsTestingUtils.writeGcsFile(gcsService, STAGE_FILE, Ghostryde.encode(xml, encryptKey)); - GcsTestingUtils.writeGcsFile(gcsService, STAGE_LENGTH_FILE, - Long.toString(xml.length).getBytes(UTF_8)); + gcsUtils.createFromBytes(STAGE_FILE, Ghostryde.encode(xml, encryptKey)); + gcsUtils.createFromBytes(STAGE_LENGTH_FILE, Long.toString(xml.length).getBytes(UTF_8)); } @Test @@ -124,7 +117,7 @@ public class BrdaCopyActionTest { action.run(); File rydeTmp = new File(gpg.getCwd(), "ryde"); - Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp); + Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp); Process pid = gpg.exec( "gpg", @@ -172,8 +165,8 @@ public class BrdaCopyActionTest { File rydeTmp = new File(gpg.getCwd(), "ryde"); File sigTmp = new File(gpg.getCwd(), "ryde.sig"); - Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp); - Files.write(readGcsFile(gcsService, SIG_FILE), sigTmp); + Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp); + Files.write(gcsUtils.readBytesFrom(SIG_FILE), sigTmp); Process pid = gpg.exec("gpg", "--verify", sigTmp.toString(), rydeTmp.toString()); String stderr = slurp(pid.getErrorStream()); diff --git a/core/src/test/java/google/registry/rde/RdeReportActionTest.java b/core/src/test/java/google/registry/rde/RdeReportActionTest.java index 81f06d72c..eda53a3de 100644 --- a/core/src/test/java/google/registry/rde/RdeReportActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeReportActionTest.java @@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.loadByKey; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.deleteGcsFile; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.joda.time.Duration.standardDays; @@ -41,13 +39,12 @@ import com.google.appengine.api.urlfetch.HTTPHeader; import com.google.appengine.api.urlfetch.HTTPRequest; import com.google.appengine.api.urlfetch.HTTPResponse; import com.google.appengine.api.urlfetch.URLFetchService; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.common.Cursor; import google.registry.model.rde.RdeRevision; import google.registry.model.registry.Registry; @@ -95,9 +92,9 @@ public class RdeReportActionTest { private final ArgumentCaptor request = ArgumentCaptor.forClass(HTTPRequest.class); private final HTTPResponse httpResponse = mock(HTTPResponse.class); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); - private final GcsFilename reportFile = - new GcsFilename("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde"); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); + private final BlobId reportFile = + BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde"); private RdeReportAction createAction() { RdeReporter reporter = new RdeReporter(); @@ -106,7 +103,7 @@ public class RdeReportActionTest { reporter.password = "foo"; reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); RdeReportAction action = new RdeReportAction(); - action.gcsUtils = new GcsUtils(gcsService, 1024); + action.gcsUtils = gcsUtils; action.response = response; action.bucket = "tub"; action.tld = "test"; @@ -126,7 +123,7 @@ public class RdeReportActionTest { Cursor.create(RDE_REPORT, DateTime.parse("2006-06-06TZ"), Registry.get("test"))); persistResource( Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test"))); - writeGcsFile(gcsService, reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0)); } @@ -169,11 +166,10 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_regeneratedReport() throws Exception { - deleteGcsFile(gcsService, reportFile); - GcsFilename newReport = - new GcsFilename("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde"); + gcsUtils.delete(reportFile); + BlobId newReport = BlobId.of("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde"); PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); - writeGcsFile(gcsService, newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1)); when(httpResponse.getResponseCode()).thenReturn(SC_OK); when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); diff --git a/core/src/test/java/google/registry/rde/RdeStagingActionTest.java b/core/src/test/java/google/registry/rde/RdeStagingActionTest.java index 01d46d084..1b9a72a87 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingActionTest.java @@ -14,7 +14,6 @@ package google.registry.rde; -import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.truth.Truth.assertThat; import static google.registry.model.common.Cursor.CursorType.BRDA; import static google.registry.model.common.Cursor.CursorType.RDE_STAGING; @@ -26,7 +25,6 @@ import static google.registry.rde.RdeFixtures.makeHostResource; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DatabaseHelper.persistResourceWithCommitLog; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.TaskQueueHelper.assertAtLeastOneTaskIsEnqueued; import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; @@ -36,16 +34,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Arrays.asList; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; -import com.google.appengine.tools.cloudstorage.ListItem; -import com.google.appengine.tools.cloudstorage.ListOptions; -import com.google.appengine.tools.cloudstorage.ListResult; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.Keyring; import google.registry.keyring.api.PgpHelper; import google.registry.model.common.Cursor; @@ -98,16 +93,16 @@ import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link RdeStagingAction}. */ public class RdeStagingActionTest extends MapreduceTestCase { - private static final GcsFilename XML_FILE = - new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde"); - private static final GcsFilename LENGTH_FILE = - new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length"); + private static final BlobId XML_FILE = + BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde"); + private static final BlobId LENGTH_FILE = + BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length"); @RegisterExtension public final InjectExtension inject = new InjectExtension(); private final FakeClock clock = new FakeClock(); private final FakeResponse response = new FakeResponse(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final List alreadyExtracted = new ArrayList<>(); private static PGPPublicKey encryptKey; @@ -131,7 +126,6 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.reducerFactory = new RdeStagingReducer.Factory(); action.reducerFactory.taskQueueUtils = new TaskQueueUtils(new Retrier(new SystemSleeper(), 1)); action.reducerFactory.lockHandler = new FakeLockHandler(true); - action.reducerFactory.gcsBufferSize = 0; action.reducerFactory.bucket = "rde-bucket"; action.reducerFactory.lockTimeout = Duration.standardHours(1); action.reducerFactory.stagingKeyBytes = PgpHelper.convertPublicKeyToBytes(encryptKey); @@ -140,6 +134,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.pendingDepositChecker.brdaInterval = Duration.standardDays(7); action.pendingDepositChecker.clock = clock; action.pendingDepositChecker.rdeInterval = Duration.standardDays(1); + action.gcsUtils = gcsUtils; action.response = response; action.transactionCooldown = Duration.ZERO; action.directory = Optional.empty(); @@ -327,7 +322,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey)); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); assertThat(header.getTld()).isEqualTo("lol"); @@ -358,7 +353,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey)); assertThat(deposit.getType()).isEqualTo(XjcRdeDepositTypeType.FULL); assertThat(deposit.getId()).isEqualTo(RdeUtil.timestampToId(DateTime.parse("2000-01-01TZ"))); assertThat(deposit.getWatermark()).isEqualTo(DateTime.parse("2000-01-01TZ")); @@ -400,7 +395,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey)); XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); @@ -494,12 +489,13 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - for (GcsFilename filename : asList( - new GcsFilename("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"), - new GcsFilename("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) { + for (BlobId filename : + asList( + BlobId.of("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"), + BlobId.of("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) { XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey)); XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); @@ -536,10 +532,10 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - GcsFilename filename = new GcsFilename("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde"); + BlobId filename = BlobId.of("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde"); XjcRdeDeposit deposit = unmarshal( - XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey)); + XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey)); XjcRdeDomain domain = extractAndRemoveContentWithType(XjcRdeDomain.class, deposit); XjcRdeIdn firstIdn = extractAndRemoveContentWithType(XjcRdeIdn.class, deposit); XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit); @@ -579,8 +575,8 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - byte[] deposit = Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey); - assertThat(Integer.parseInt(new String(readGcsFile(gcsService, LENGTH_FILE), UTF_8))) + byte[] deposit = Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey); + assertThat(Integer.parseInt(new String(gcsUtils.readBytesFrom(LENGTH_FILE), UTF_8))) .isEqualTo(deposit.length); } @@ -781,23 +777,20 @@ public class RdeStagingActionTest extends MapreduceTestCase { action.run(); executeTasksUntilEmpty("mapreduce", clock); - ListResult listResult = - gcsService.list("rde-bucket", new ListOptions.Builder().setPrefix("manual/test").build()); - ImmutableSet filenames = - ImmutableList.copyOf(listResult).stream().map(ListItem::getName).collect(toImmutableSet()); + ImmutableList filenames = gcsUtils.listFolderObjects("rde-bucket", "manual/test/"); for (String tld : tlds) { assertThat(filenames) .containsAtLeast( - "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde", - "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length", - "manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length", - "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde", - "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length", - "manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde", - "manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length"); + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde", + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length", + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length", + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde", + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length", + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde", + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length"); assertThat( auditedOfy() @@ -837,8 +830,8 @@ public class RdeStagingActionTest extends MapreduceTestCase { } private String readXml(String objectName) throws IOException, PGPException { - GcsFilename file = new GcsFilename("rde-bucket", objectName); - return new String(Ghostryde.decode(readGcsFile(gcsService, file), decryptKey), UTF_8); + BlobId file = BlobId.of("rde-bucket", objectName); + return new String(Ghostryde.decode(gcsUtils.readBytesFrom(file), decryptKey), UTF_8); } private diff --git a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java index 3014091f9..f2fa21bef 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java @@ -20,18 +20,18 @@ import static google.registry.model.rde.RdeMode.FULL; import static google.registry.model.rde.RdeMode.THIN; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; import static google.registry.util.ResourceUtils.readResourceUtf8; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; import com.google.appengine.tools.mapreduce.ReducerInput; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.StorageException; import com.google.common.collect.ImmutableList; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.PgpHelper; import google.registry.model.common.Cursor; import google.registry.model.common.Cursor.CursorType; @@ -67,7 +67,7 @@ class RdeStagingReducerTest { AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); private static final String GCS_BUCKET = "test-rde-bucket"; - private static final GcsService gcsService = GcsServiceFactory.createGcsService(); + private static final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private static final PGPPrivateKey decryptionKey = new FakeKeyringModule().get().getRdeStagingDecryptionKey(); private static final PGPPublicKey encryptionKey = @@ -95,11 +95,11 @@ class RdeStagingReducerTest { new RdeStagingReducer( new TaskQueueUtils(new Retrier(new FakeSleeper(new FakeClock()), 1)), new FakeLockHandler(true), - 1024, GCS_BUCKET, Duration.ZERO, PgpHelper.convertPublicKeyToBytes(encryptionKey), - ValidationMode.STRICT); + ValidationMode.STRICT, + gcsUtils); @BeforeEach void beforeEach() { @@ -125,11 +125,10 @@ class RdeStagingReducerTest { compareLength(outputFile, "soy_2000-01-01_thin_S1_R1.xml.length"); // BRDA doesn't write a report file. assertThrows( - IOException.class, + StorageException.class, () -> - readGcsFile( - gcsService, - new GcsFilename(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde"))); + gcsUtils.readBytesFrom( + BlobId.of(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde"))); assertThat(loadCursorTime(CursorType.BRDA)) .isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1))); assertThat(loadRevision(THIN)).isEqualTo(1); @@ -153,12 +152,10 @@ class RdeStagingReducerTest { compareLength(outputFile, "manual/soy_2000-01-01_thin_S1_R0.xml.length"); // BRDA doesn't write a report file. assertThrows( - IOException.class, + StorageException.class, () -> - readGcsFile( - gcsService, - new GcsFilename( - GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde"))); + gcsUtils.readBytesFrom( + BlobId.of(GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde"))); // No extra operations in manual mode. assertThat(loadCursorTime(CursorType.BRDA)).isEquivalentAccordingToCompareTo(now); assertThat(loadRevision(THIN)).isEqualTo(0); @@ -210,8 +207,7 @@ class RdeStagingReducerTest { private static void compareLength(String outputFile, String lengthFilename) throws IOException { assertThat(String.valueOf(outputFile.getBytes(UTF_8).length)) .isEqualTo( - new String( - readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, lengthFilename)), UTF_8)); + new String(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, lengthFilename)), UTF_8)); } private static DateTime loadCursorTime(CursorType type) { @@ -233,8 +229,7 @@ class RdeStagingReducerTest { private static String decryptGhostrydeGcsFile(String filename) throws IOException, PGPException { return new String( - Ghostryde.decode( - readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, filename)), decryptionKey), + Ghostryde.decode(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, filename)), decryptionKey), UTF_8); } diff --git a/core/src/test/java/google/registry/rde/RdeUploadActionTest.java b/core/src/test/java/google/registry/rde/RdeUploadActionTest.java index 1281ed920..271f669d6 100644 --- a/core/src/test/java/google/registry/rde/RdeUploadActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeUploadActionTest.java @@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DatabaseHelper.persistSimpleResource; -import static google.registry.testing.GcsTestingUtils.readGcsFile; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static google.registry.testing.SystemInfo.hasCommand; import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued; import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued; @@ -45,9 +43,7 @@ import static org.mockito.Mockito.when; import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.utils.SystemProperty; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.io.ByteSource; import com.google.common.io.CharStreams; import com.google.common.io.Files; @@ -55,6 +51,7 @@ import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.keyring.api.Keyring; import google.registry.model.common.Cursor; import google.registry.model.common.Cursor.CursorType; @@ -94,23 +91,24 @@ import org.mockito.stubbing.OngoingStubbing; @DualDatabaseTest public class RdeUploadActionTest { - private static final int BUFFER_SIZE = 64 * 1024; private static final ByteSource REPORT_XML = RdeTestData.loadBytes("report.xml"); private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml"); - private static final GcsFilename GHOSTRYDE_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde"); - private static final GcsFilename LENGTH_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.length"); - private static final GcsFilename REPORT_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde"); + private static final BlobId GHOSTRYDE_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde"); + private static final BlobId LENGTH_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.length"); + private static final BlobId REPORT_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde"); - private static final GcsFilename GHOSTRYDE_R1_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde"); - private static final GcsFilename LENGTH_R1_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.length"); - private static final GcsFilename REPORT_R1_FILE = - new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde"); + private static final BlobId GHOSTRYDE_R1_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde"); + private static final BlobId LENGTH_R1_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.length"); + private static final BlobId REPORT_R1_FILE = + BlobId.of("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde"); + + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @RegisterExtension final SftpServerExtension sftpd = new SftpServerExtension(); @@ -139,7 +137,7 @@ public class RdeUploadActionTest { try (Keyring keyring = new FakeKeyringModule().get()) { RdeUploadAction action = new RdeUploadAction(); action.clock = clock; - action.gcsUtils = new GcsUtils(gcsService, BUFFER_SIZE); + action.gcsUtils = gcsUtils; action.lazyJsch = () -> JSchModule.provideJSch( @@ -176,22 +174,20 @@ public class RdeUploadActionTest { return jschSpy; } - private GcsService gcsService; @BeforeEach void beforeEach() throws Exception { // Force "development" mode so we don't try to really connect to GCS. SystemProperty.environment.set(SystemProperty.Environment.Value.Development); - gcsService = GcsServiceFactory.createGcsService(); createTld("tld"); PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); - writeGcsFile(gcsService, GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); - writeGcsFile(gcsService, GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); - writeGcsFile(gcsService, LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); - writeGcsFile(gcsService, LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); - writeGcsFile(gcsService, REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); - writeGcsFile(gcsService, REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); + gcsUtils.createFromBytes(LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8)); + gcsUtils.createFromBytes(REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); + gcsUtils.createFromBytes(REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm() .transact( () -> { @@ -271,9 +267,9 @@ public class RdeUploadActionTest { String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde"; String sigFilename = "tld_2010-10-17_full_S1_R0.sig"; assertThat(folder.list()).asList().containsExactly(rydeFilename, sigFilename); - assertThat(readGcsFile(gcsService, new GcsFilename("bucket", rydeFilename))) + assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", rydeFilename))) .isEqualTo(Files.toByteArray(new File(folder, rydeFilename))); - assertThat(readGcsFile(gcsService, new GcsFilename("bucket", sigFilename))) + assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", sigFilename))) .isEqualTo(Files.toByteArray(new File(folder, sigFilename))); } diff --git a/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java b/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java index b4089250c..e43dbd22f 100644 --- a/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java +++ b/core/src/test/java/google/registry/reporting/billing/BillingEmailUtilsTest.java @@ -23,7 +23,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.net.MediaType; import google.registry.gcs.GcsUtils; @@ -51,7 +51,7 @@ class BillingEmailUtilsTest { void beforeEach() throws Exception { emailService = mock(SendEmailService.class); gcsUtils = mock(GcsUtils.class); - when(gcsUtils.openInputStream(new GcsFilename("test-bucket", "results/REG-INV-2017-10.csv"))) + when(gcsUtils.openInputStream(BlobId.of("test-bucket", "results/REG-INV-2017-10.csv"))) .thenReturn( new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8))); contentCaptor = ArgumentCaptor.forClass(EmailMessage.class); diff --git a/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java b/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java index 4b63f8170..fc2d79f7a 100644 --- a/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java +++ b/core/src/test/java/google/registry/reporting/billing/CopyDetailReportsActionTest.java @@ -17,7 +17,6 @@ package google.registry.reporting.billing; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatabaseHelper.loadRegistrar; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.mockito.ArgumentMatchers.any; @@ -29,11 +28,10 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.net.MediaType; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.storage.drive.DriveConnection; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; @@ -52,8 +50,7 @@ class CopyDetailReportsActionTest { final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); - private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private FakeResponse response; private DriveConnection driveConnection; @@ -80,14 +77,12 @@ class CopyDetailReportsActionTest { @Test void testSuccess() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"), "hello,world\n1,2".getBytes(UTF_8)); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); action.run(); @@ -111,14 +106,12 @@ class CopyDetailReportsActionTest { @Test void testSuccess_nonDetailReportFiles_notSent() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"), "hello,world\n1,2".getBytes(UTF_8)); action.run(); verify(driveConnection) @@ -136,9 +129,8 @@ class CopyDetailReportsActionTest { @Test void testSuccess_transientIOException_retries() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); when(driveConnection.createOrUpdateFile(any(), any(), any(), any())) .thenThrow(new IOException("expected")) @@ -158,13 +150,12 @@ class CopyDetailReportsActionTest { @Test void testFail_tooManyFailures_sendsAlertEmail_continues() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"), + + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"), "hello,world\n1,2".getBytes(UTF_8)); when(driveConnection.createOrUpdateFile( eq("invoice_details_2017-10_TheRegistrar_hello.csv"), any(), any(), any())) @@ -201,9 +192,8 @@ class CopyDetailReportsActionTest { @Test void testFail_registrarDoesntExist_doesntCopy() throws IOException { - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); action.run(); verifyNoInteractions(driveConnection); @@ -212,9 +202,8 @@ class CopyDetailReportsActionTest { @Test void testFail_noRegistrarFolderId_doesntCopy() throws IOException { persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId(null).build()); - writeGcsFile( - gcsService, - new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), + gcsUtils.createFromBytes( + BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"), "hola,mundo\n3,4".getBytes(UTF_8)); action.run(); verifyNoInteractions(driveConnection); diff --git a/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java b/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java index edf2ede40..716ece531 100644 --- a/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java +++ b/core/src/test/java/google/registry/reporting/icann/IcannReportingStagerTest.java @@ -15,16 +15,13 @@ package google.registry.reporting.icann; import static com.google.common.truth.Truth.assertThat; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static java.nio.charset.StandardCharsets.UTF_8; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableTable; import com.google.common.util.concurrent.ListenableFuture; @@ -32,6 +29,7 @@ import google.registry.bigquery.BigqueryConnection; import google.registry.bigquery.BigqueryConnection.DestinationTable; import google.registry.bigquery.BigqueryUtils.TableType; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.reporting.icann.IcannReportingModule.ReportType; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeResponse; @@ -46,9 +44,9 @@ class IcannReportingStagerTest { private BigqueryConnection bigquery = mock(BigqueryConnection.class); FakeResponse response = new FakeResponse(); - private GcsService gcsService = GcsServiceFactory.createGcsService(); private YearMonth yearMonth = new YearMonth(2017, 6); private String subdir = "icann/monthly/2017-06"; + private GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @RegisterExtension final AppEngineExtension appEngine = @@ -65,7 +63,7 @@ class IcannReportingStagerTest { action.transactionsQueryBuilder = transactionsBuilder; action.reportingBucket = "test-bucket"; action.bigquery = bigquery; - action.gcsUtils = new GcsUtils(gcsService, 1024); + action.gcsUtils = gcsUtils; return action; } @@ -100,14 +98,12 @@ class IcannReportingStagerTest { String expectedReport1 = "fooField,barField\r\n12,34"; String expectedReport2 = "fooField,barField\r\n56,78"; byte[] generatedFile1 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv")); assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1); byte[] generatedFile2 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv")); assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2); } @@ -144,14 +140,12 @@ class IcannReportingStagerTest { "registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30"; String expectedReport2 = "registrar,iana,field\r\n\"reg1\",123,30\r\nTotals,,30"; byte[] generatedFile1 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv")); assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1); byte[] generatedFile2 = - readGcsFile( - gcsService, - new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv")); + gcsUtils.readBytesFrom( + BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv")); assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2); } @@ -164,8 +158,7 @@ class IcannReportingStagerTest { String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n"; byte[] generatedManifest = - readGcsFile( - gcsService, new GcsFilename("test-bucket/icann/monthly/2017-06", "MANIFEST.txt")); + gcsUtils.readBytesFrom(BlobId.of("test-bucket/icann/monthly/2017-06", "MANIFEST.txt")); assertThat(new String(generatedManifest, UTF_8)).isEqualTo(expectedManifest); } diff --git a/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java b/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java index 3c4203975..68be3ed91 100644 --- a/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java +++ b/core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java @@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.createTlds; import static google.registry.testing.DatabaseHelper.loadByKey; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.writeGcsFile; import static google.registry.testing.LogsSubject.assertAboutLogs; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -29,11 +28,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.cloud.storage.BlobId; import com.google.common.testing.TestLogHandler; import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.common.Cursor; import google.registry.model.common.Cursor.CursorType; import google.registry.model.registry.Registry; @@ -69,7 +67,7 @@ class IcannReportingUploadActionTest { private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class); private final SendEmailService emailService = mock(SendEmailService.class); private final FakeResponse response = new FakeResponse(); - private final GcsService gcsService = GcsServiceFactory.createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final TestLogHandler logHandler = new TestLogHandler(); private final Logger loggerToIntercept = Logger.getLogger(IcannReportingUploadAction.class.getCanonicalName()); @@ -78,7 +76,7 @@ class IcannReportingUploadActionTest { private IcannReportingUploadAction createAction() throws Exception { IcannReportingUploadAction action = new IcannReportingUploadAction(); action.icannReporter = mockReporter; - action.gcsUtils = new GcsUtils(gcsService, 1024); + action.gcsUtils = gcsUtils; action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); action.reportingBucket = "basin"; action.emailService = emailService; @@ -93,22 +91,14 @@ class IcannReportingUploadActionTest { @BeforeEach void beforeEach() throws Exception { createTlds("tld", "foo"); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"), - PAYLOAD_SUCCESS); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), - PAYLOAD_FAIL); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"), - PAYLOAD_SUCCESS); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "foo-activity-200606.csv"), - PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"), PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_FAIL); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"), PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "foo-activity-200606.csv"), PAYLOAD_SUCCESS); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv")).thenReturn(true); when(mockReporter.send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv")).thenReturn(true); when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv")).thenReturn(false); @@ -161,14 +151,10 @@ class IcannReportingUploadActionTest { persistResource( Cursor.create( CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-01-01TZ"), Registry.get("tld"))); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"), - PAYLOAD_SUCCESS); - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2005-12", "tld-activity-200512.csv"), - PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"), PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2005-12", "tld-activity-200512.csv"), PAYLOAD_SUCCESS); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200512.csv")).thenReturn(true); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200512.csv")).thenReturn(true); @@ -191,10 +177,8 @@ class IcannReportingUploadActionTest { @TestOfyAndSql void testSuccess_advancesCursor() throws Exception { - writeGcsFile( - gcsService, - new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), - PAYLOAD_SUCCESS); + gcsUtils.createFromBytes( + BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_SUCCESS); when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true); IcannReportingUploadAction action = createAction(); action.run(); diff --git a/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java b/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java index 4d7136646..c0702d0e7 100644 --- a/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java +++ b/core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java @@ -19,7 +19,7 @@ import static com.google.common.truth.Truth8.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.appengine.tools.cloudstorage.GcsFilename; +import com.google.cloud.storage.BlobId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -133,8 +133,8 @@ public class Spec11RegistrarThreatMatchesParserTest { } private void setupFile(String fileWithContent, String fileDate) { - GcsFilename gcsFilename = - new GcsFilename( + BlobId gcsFilename = + BlobId.of( "test-bucket", String.format("icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_%s", fileDate)); when(gcsUtils.existsAndNotEmpty(gcsFilename)).thenReturn(true); diff --git a/core/src/test/java/google/registry/testing/GcsTestingUtils.java b/core/src/test/java/google/registry/testing/GcsTestingUtils.java deleted file mode 100644 index a51bb3dfe..000000000 --- a/core/src/test/java/google/registry/testing/GcsTestingUtils.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.testing; - -import com.google.appengine.tools.cloudstorage.GcsFileOptions; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.common.io.ByteStreams; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; - -/** Utility methods for testing Google Cloud Storage code. */ -public final class GcsTestingUtils { - - /** Slurps a Cloud Storage file into memory. */ - public static byte[] readGcsFile(GcsService gcsService, GcsFilename file) - throws IOException { - try (InputStream input = Channels.newInputStream(gcsService.openReadChannel(file, 0))) { - return ByteStreams.toByteArray(input); - } - } - - /** Writes a Cloud Storage file. */ - public static void writeGcsFile(GcsService gcsService, GcsFilename file, byte[] data) - throws IOException { - gcsService.createOrReplace(file, GcsFileOptions.getDefaultInstance(), ByteBuffer.wrap(data)); - } - - public static void deleteGcsFile(GcsService gcsService, GcsFilename file) throws IOException { - gcsService.delete(file); - } - - private GcsTestingUtils() {} -} diff --git a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java index c65a02864..d771d5e0e 100644 --- a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java @@ -14,7 +14,6 @@ package google.registry.tools.server; -import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatabaseHelper.createTlds; import static google.registry.testing.DatabaseHelper.newDomainBase; @@ -23,17 +22,17 @@ import static google.registry.testing.DatabaseHelper.persistActiveContact; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistActiveHost; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.GcsTestingUtils.readGcsFile; import static google.registry.testing.TestDataHelper.loadFile; import static java.nio.charset.StandardCharsets.UTF_8; import static org.joda.time.Duration.standardDays; -import com.google.appengine.tools.cloudstorage.GcsFilename; -import com.google.appengine.tools.cloudstorage.GcsService; +import com.google.cloud.storage.BlobId; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import google.registry.gcs.GcsUtils; +import google.registry.gcs.backport.LocalStorageHelper; import google.registry.model.domain.secdns.DelegationSignerData; import google.registry.model.eppcommon.StatusValue; import google.registry.model.host.HostResource; @@ -50,7 +49,7 @@ import org.junit.jupiter.api.Test; /** Tests for {@link GenerateZoneFilesAction}. */ class GenerateZoneFilesActionTest extends MapreduceTestCase { - private final GcsService gcsService = createGcsService(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @Test void testGenerate() throws Exception { @@ -114,7 +113,7 @@ class GenerateZoneFilesActionTest extends MapreduceTestCase