mirror of
https://github.com/google/nomulus.git
synced 2025-04-29 19:47:51 +02:00
Remove the use of GCS APIs provided from GAE SDK (#1228)
The API provided by the GAE SDK will not be available outside GAE runtime. This presents a problem when we migrate off of GAE. More pressingly, the RDE pipeline migration to Beam requires that we write to GCS on GCE. Previously we were able to sidestep the issue by delegating the writes to FileIO provided by Beam, which knows how to write to GCS. However the RDE pipeline cannot use FileIO directly as it needs to write to multiple files in one go and explicit use of GCS API is needed. An unfortunate side effect of the API migration is that the new testing library contains a bug which makes serializing GcsUtils impossible. It is fixed upstream but not released yet. The fix has been backported for the time being. <!-- Reviewable:start --> This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1228) <!-- Reviewable:end -->
This commit is contained in:
Lai Jiang
GitHub
parent
b747b5aca9
commit
65ae5f4a30
186 changed files with 2415 additions and 1600 deletions
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
|||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
|||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
|||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
|||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
@ -53,7 +53,7 @@ org.apache.commons:commons-text:1.6
|
|||
org.apache.httpcomponents:httpclient:4.5.13
|
||||
org.apache.httpcomponents:httpcore:4.4.14
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.json:json:20160212
|
||||
org.ow2.asm:asm-analysis:7.0
|
||||
org.ow2.asm:asm-commons:7.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ com.google.common.html.types:types:1.0.6
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
|
|
|||
|
|
@ -2,11 +2,11 @@
|
|||
# Manual edits can break the build and are not advised.
|
||||
# This file is expected to be part of source control.
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
javax.inject:javax.inject:1
|
||||
joda-time:joda-time:2.9.2
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
|||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
|||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
|||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
|||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -79,8 +79,8 @@ PRESUBMITS = {
|
|||
r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
|
||||
("java", "js", "soy", "sql", "py", "sh", "gradle"), {
|
||||
".git", "/build/", "/generated/", "/generated_tests/",
|
||||
"node_modules/", "JUnitBackports.java", "registrar_bin.",
|
||||
"registrar_dbg.", "google-java-format-diff.py",
|
||||
"node_modules/", "LocalStorageHelper.java", "FakeStorageRpc.java",
|
||||
"registrar_bin.", "registrar_dbg.", "google-java-format-diff.py",
|
||||
"nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
|
||||
}, REQUIRED):
|
||||
"File did not include the license header.",
|
||||
|
|
|
|||
|
|
@ -197,6 +197,7 @@ dependencies {
|
|||
compile deps['com.google.apis:google-api-services-groupssettings']
|
||||
compile deps['com.google.apis:google-api-services-monitoring']
|
||||
compile deps['com.google.apis:google-api-services-sheets']
|
||||
compile deps['com.google.apis:google-api-services-storage']
|
||||
testCompile deps['com.google.appengine:appengine-api-stubs']
|
||||
compile deps['com.google.appengine.tools:appengine-gcs-client']
|
||||
compile deps['com.google.appengine.tools:appengine-mapreduce']
|
||||
|
|
@ -221,6 +222,8 @@ dependencies {
|
|||
gradleLint.ignore('unused-dependency') {
|
||||
compile deps['com.google.gwt:gwt-user']
|
||||
}
|
||||
compile deps['com.google.cloud:google-cloud-core']
|
||||
compile deps['com.google.cloud:google-cloud-storage']
|
||||
compile deps['com.google.http-client:google-http-client']
|
||||
compile deps['com.google.http-client:google-http-client-appengine']
|
||||
compile deps['com.google.http-client:google-http-client-jackson2']
|
||||
|
|
@ -315,6 +318,7 @@ dependencies {
|
|||
annotationProcessor project(':processor')
|
||||
testAnnotationProcessor project(':processor')
|
||||
|
||||
testCompile deps['com.google.cloud:google-cloud-nio']
|
||||
testCompile deps['com.google.appengine:appengine-testing']
|
||||
testCompile deps['com.google.guava:guava-testlib']
|
||||
testCompile deps['com.google.monitoring-client:contrib']
|
||||
|
|
|
|||
|
|
@ -14,14 +14,14 @@ com.google.dagger:dagger-producers:2.33
|
|||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
|
|
@ -32,7 +32,7 @@ javax.inject:javax.inject:1
|
|||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
|
|
|||
|
|
@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -215,7 +216,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -55,10 +55,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -97,12 +97,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -113,7 +114,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -133,8 +134,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -221,7 +222,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -111,7 +112,7 @@ com.google.escapevelocity:escapevelocity:0.9.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -131,8 +132,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -216,7 +217,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -101,12 +101,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -117,7 +118,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -137,8 +138,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ com.google.dagger:dagger-producers:2.33
|
|||
com.google.dagger:dagger-spi:2.33
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotation:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_check_api:2.3.4
|
||||
com.google.errorprone:error_prone_core:2.3.4
|
||||
com.google.errorprone:error_prone_type_annotations:2.3.4
|
||||
com.google.errorprone:javac-shaded:9-dev-r4023-3
|
||||
com.google.googlejavaformat:google-java-format:1.5
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.protobuf:protobuf-java:3.4.0
|
||||
|
|
@ -30,7 +30,7 @@ javax.inject:javax.inject:1
|
|||
javax.persistence:javax.persistence-api:2.2
|
||||
net.ltgt.gradle.incap:incap:0.2
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.checkerframework:dataflow:3.0.0
|
||||
org.checkerframework:javacutil:3.0.0
|
||||
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.20
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
|||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
|
|
@ -28,7 +28,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
|||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
|
|
@ -54,12 +54,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
|
|
@ -76,7 +76,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -84,10 +84,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
|||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
|
|
@ -98,32 +98,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
|
|
@ -135,9 +137,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
|
|
@ -161,7 +163,7 @@ io.github.classgraph:classgraph:4.8.102
|
|||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
|
@ -199,7 +201,7 @@ javax.validation:validation-api:1.0.0.GA
|
|||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
|
|
@ -307,7 +309,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
|||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ aopalliance:aopalliance:1.0
|
|||
args4j:args4j:2.0.23
|
||||
cglib:cglib-nodep:2.2
|
||||
com.beust:jcommander:1.60
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
|
|
@ -27,7 +27,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
|||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
|
|
@ -53,12 +53,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
|
|
@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -83,10 +83,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
|||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
|
|
@ -97,31 +97,33 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
|
|
@ -133,9 +135,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
|
|
@ -159,7 +161,7 @@ io.github.classgraph:classgraph:4.8.102
|
|||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
|
@ -194,7 +196,7 @@ javax.validation:validation-api:1.0.0.GA
|
|||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
net.java.dev.jna:jna:5.5.0
|
||||
|
|
@ -301,7 +303,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
|||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.xerial.snappy:snappy-java:1.1.4
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
|||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
|
|
@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
|||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
|
|
@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
|||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
|
|
@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
|
|
@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
|
|
@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
|||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
|
@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
|
|||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
|
|
@ -320,7 +322,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
|||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ com.eclipsesource.j2v8:j2v8_linux_x86_64:4.6.0
|
|||
com.eclipsesource.j2v8:j2v8_macosx_x86_64:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86:4.6.0
|
||||
com.eclipsesource.j2v8:j2v8_win32_x86_64:4.6.0
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.1
|
||||
com.fasterxml.jackson:jackson-bom:2.12.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-core:2.12.3
|
||||
com.fasterxml.jackson.core:jackson-databind:2.12.3
|
||||
com.fasterxml.jackson:jackson-bom:2.12.3
|
||||
com.fasterxml:classmate:1.5.1
|
||||
com.github.docker-java:docker-java-api:3.2.7
|
||||
com.github.docker-java:docker-java-transport-zerodep:3.2.7
|
||||
|
|
@ -32,7 +32,7 @@ com.google.api-client:google-api-client-appengine:1.31.3
|
|||
com.google.api-client:google-api-client-jackson2:1.30.10
|
||||
com.google.api-client:google-api-client-java6:1.31.3
|
||||
com.google.api-client:google-api-client-servlet:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.3
|
||||
com.google.api-client:google-api-client:1.31.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:1.5.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.105.5
|
||||
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.105.5
|
||||
|
|
@ -58,12 +58,12 @@ com.google.api.grpc:proto-google-cloud-secretmanager-v1beta1:1.4.0
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api.grpc:proto-google-common-protos:2.3.2
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.14
|
||||
com.google.api:api-common:1.10.3
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.api:gax-httpjson:0.81.0
|
||||
com.google.api:gax:1.65.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
com.google.apis:google-api-services-bigquery:v2-rev20200916-1.30.10
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -88,10 +88,10 @@ com.google.appengine:appengine-api-1.0-sdk:1.9.86
|
|||
com.google.appengine:appengine-api-stubs:1.9.86
|
||||
com.google.appengine:appengine-remote-api:1.9.86
|
||||
com.google.appengine:appengine-testing:1.9.86
|
||||
com.google.auth:google-auth-library-credentials:0.24.1
|
||||
com.google.auth:google-auth-library-oauth2-http:0.24.1
|
||||
com.google.auth:google-auth-library-credentials:0.26.0
|
||||
com.google.auth:google-auth-library-oauth2-http:0.26.0
|
||||
com.google.auto.service:auto-service-annotations:1.0-rc7
|
||||
com.google.auto.value:auto-value-annotations:1.7.4
|
||||
com.google.auto.value:auto-value-annotations:1.8.1
|
||||
com.google.auto.value:auto-value:1.7.4
|
||||
com.google.cloud.bigdataoss:gcsio:2.1.6
|
||||
com.google.cloud.bigdataoss:util:2.1.6
|
||||
|
|
@ -103,32 +103,34 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.8
|
||||
com.google.cloud:google-cloud-core:1.95.0
|
||||
com.google.cloud:google-cloud-nio:0.123.2
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.115.0
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.errorprone:error_prone_annotations:2.7.1
|
||||
com.google.escapevelocity:escapevelocity:0.9.1
|
||||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava-testlib:30.1-jre
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava-testlib:30.1.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-appengine:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
com.google.http-client:google-http-client-jackson2:1.39.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.2
|
||||
com.google.http-client:google-http-client-appengine:1.39.2
|
||||
com.google.http-client:google-http-client-gson:1.39.2
|
||||
com.google.http-client:google-http-client-jackson2:1.39.2
|
||||
com.google.http-client:google-http-client-protobuf:1.33.0
|
||||
com.google.http-client:google-http-client:1.39.0
|
||||
com.google.http-client:google-http-client:1.39.2
|
||||
com.google.inject.extensions:guice-multibindings:4.1.0
|
||||
com.google.inject:guice:4.1.0
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
|
|
@ -140,9 +142,9 @@ com.google.oauth-client:google-oauth-client-appengine:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-java6:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.oauth-client:google-oauth-client:1.31.5
|
||||
com.google.protobuf:protobuf-java-util:3.17.2
|
||||
com.google.protobuf:protobuf-java:3.17.2
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth.extensions:truth-java8-extension:1.1.2
|
||||
|
|
@ -170,7 +172,7 @@ io.github.java-diff-utils:java-diff-utils:4.9
|
|||
io.grpc:grpc-alts:1.36.0
|
||||
io.grpc:grpc-api:1.36.0
|
||||
io.grpc:grpc-auth:1.36.0
|
||||
io.grpc:grpc-context:1.36.0
|
||||
io.grpc:grpc-context:1.37.1
|
||||
io.grpc:grpc-core:1.36.0
|
||||
io.grpc:grpc-grpclb:1.36.0
|
||||
io.grpc:grpc-netty-shaded:1.36.0
|
||||
|
|
@ -208,7 +210,7 @@ javax.validation:validation-api:1.0.0.GA
|
|||
javax.xml.bind:jaxb-api:2.3.1
|
||||
jline:jline:1.0
|
||||
joda-time:joda-time:2.10.5
|
||||
junit:junit:4.13.1
|
||||
junit:junit:4.13.2
|
||||
net.arnx:nashorn-promise:0.1.1
|
||||
net.bytebuddy:byte-buddy-agent:1.10.19
|
||||
net.bytebuddy:byte-buddy:1.10.19
|
||||
|
|
@ -321,7 +323,7 @@ org.testcontainers:junit-jupiter:1.15.2
|
|||
org.testcontainers:postgresql:1.15.2
|
||||
org.testcontainers:selenium:1.15.2
|
||||
org.testcontainers:testcontainers:1.15.2
|
||||
org.threeten:threetenbp:1.5.0
|
||||
org.threeten:threetenbp:1.5.1
|
||||
org.tukaani:xz:1.5
|
||||
org.w3c.css:sac:1.3
|
||||
org.webjars.npm:viz.js-for-graphviz-java:2.1.3
|
||||
|
|
|
|||
|
|
@ -28,18 +28,17 @@ import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
|
|||
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.isAtOrAfter;
|
||||
import static java.nio.channels.Channels.newOutputStream;
|
||||
import static java.util.Comparator.comparingLong;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
|
|
@ -74,7 +73,8 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
|||
|
||||
public static final String DIFF_FILE_PREFIX = "commit_diff_until_";
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
@Inject @Config("commitLogDiffExportBatchSize") int batchSize;
|
||||
@Inject @Parameter(LOWER_CHECKPOINT_TIME_PARAM) DateTime lowerCheckpointTime;
|
||||
|
|
@ -102,13 +102,13 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
|||
List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
|
||||
logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
|
||||
// Open an output channel to GCS, wrapped in a stream for convenience.
|
||||
try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
|
||||
new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString())
|
||||
.addUserMetadata(UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString())
|
||||
.addUserMetadata(NUM_TRANSACTIONS, Integer.toString(sortedKeys.size()))
|
||||
.build()))) {
|
||||
try (OutputStream gcsStream =
|
||||
gcsUtils.openOutputStream(
|
||||
BlobId.of(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
ImmutableMap.of(
|
||||
LOWER_BOUND_CHECKPOINT, lowerCheckpointTime.toString(),
|
||||
UPPER_BOUND_CHECKPOINT, upperCheckpointTime.toString(),
|
||||
NUM_TRANSACTIONS, Integer.toString(sortedKeys.size())))) {
|
||||
// Export the upper checkpoint itself.
|
||||
serializeEntity(upperCheckpoint, gcsStream);
|
||||
// If there are no manifests to export, stop early, now that we've written out the file with
|
||||
|
|
|
|||
|
|
@ -21,19 +21,16 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
|||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
import static google.registry.util.DateTimeUtils.latestOf;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
import google.registry.backup.BackupModule.Backups;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import javax.annotation.Nullable;
|
||||
|
|
@ -45,7 +42,8 @@ class GcsDiffFileLister {
|
|||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Backups ListeningExecutorService executor;
|
||||
@Inject GcsDiffFileLister() {}
|
||||
|
||||
|
|
@ -56,22 +54,22 @@ class GcsDiffFileLister {
|
|||
*/
|
||||
private boolean constructDiffSequence(
|
||||
String gcsBucket,
|
||||
Map<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata,
|
||||
Map<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo,
|
||||
DateTime fromTime,
|
||||
DateTime lastTime,
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence) {
|
||||
TreeMap<DateTime, BlobInfo> sequence) {
|
||||
DateTime checkpointTime = lastTime;
|
||||
while (isBeforeOrAt(fromTime, checkpointTime)) {
|
||||
GcsFileMetadata metadata;
|
||||
if (upperBoundTimesToMetadata.containsKey(checkpointTime)) {
|
||||
metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
|
||||
BlobInfo blobInfo;
|
||||
if (upperBoundTimesToBlobInfo.containsKey(checkpointTime)) {
|
||||
blobInfo = Futures.getUnchecked(upperBoundTimesToBlobInfo.get(checkpointTime));
|
||||
} else {
|
||||
String filename = DIFF_FILE_PREFIX + checkpointTime;
|
||||
logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
|
||||
metadata = getMetadata(gcsBucket, filename);
|
||||
blobInfo = getBlobInfo(gcsBucket, filename);
|
||||
|
||||
// If we hit a gap, quit.
|
||||
if (metadata == null) {
|
||||
if (blobInfo == null) {
|
||||
logger.atInfo().log(
|
||||
"Gap discovered in sequence terminating at %s, missing file: %s",
|
||||
sequence.lastKey(), filename);
|
||||
|
|
@ -79,14 +77,14 @@ class GcsDiffFileLister {
|
|||
return false;
|
||||
}
|
||||
}
|
||||
sequence.put(checkpointTime, metadata);
|
||||
checkpointTime = getLowerBoundTime(metadata);
|
||||
sequence.put(checkpointTime, blobInfo);
|
||||
checkpointTime = getLowerBoundTime(blobInfo);
|
||||
}
|
||||
logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
return true;
|
||||
}
|
||||
|
||||
ImmutableList<GcsFileMetadata> listDiffFiles(
|
||||
ImmutableList<BlobInfo> listDiffFiles(
|
||||
String gcsBucket, DateTime fromTime, @Nullable DateTime toTime) {
|
||||
logger.atInfo().log("Requested restore from time: %s", fromTime);
|
||||
if (toTime != null) {
|
||||
|
|
@ -95,28 +93,25 @@ class GcsDiffFileLister {
|
|||
// List all of the diff files on GCS and build a map from each file's upper checkpoint time
|
||||
// (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
|
||||
// an upper checkpoint time > fromTime.
|
||||
TreeMap<DateTime, ListenableFuture<GcsFileMetadata>> upperBoundTimesToMetadata
|
||||
= new TreeMap<>();
|
||||
Iterator<ListItem> listItems;
|
||||
TreeMap<DateTime, ListenableFuture<BlobInfo>> upperBoundTimesToBlobInfo = new TreeMap<>();
|
||||
ImmutableList<String> strippedFilenames;
|
||||
try {
|
||||
// TODO(b/23554360): Use a smarter prefixing strategy to speed this up.
|
||||
listItems = gcsService.list(
|
||||
gcsBucket,
|
||||
new ListOptions.Builder().setPrefix(DIFF_FILE_PREFIX).build());
|
||||
strippedFilenames = gcsUtils.listFolderObjects(gcsBucket, DIFF_FILE_PREFIX);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
DateTime lastUpperBoundTime = START_OF_TIME;
|
||||
while (listItems.hasNext()) {
|
||||
final String filename = listItems.next().getName();
|
||||
DateTime upperBoundTime = DateTime.parse(filename.substring(DIFF_FILE_PREFIX.length()));
|
||||
|
||||
for (String strippedFilename : strippedFilenames) {
|
||||
final String filename = DIFF_FILE_PREFIX + strippedFilename;
|
||||
DateTime upperBoundTime = DateTime.parse(strippedFilename);
|
||||
if (isInRange(upperBoundTime, fromTime, toTime)) {
|
||||
upperBoundTimesToMetadata.put(
|
||||
upperBoundTime, executor.submit(() -> getMetadata(gcsBucket, filename)));
|
||||
upperBoundTimesToBlobInfo.put(
|
||||
upperBoundTime, executor.submit(() -> getBlobInfo(gcsBucket, filename)));
|
||||
lastUpperBoundTime = latestOf(upperBoundTime, lastUpperBoundTime);
|
||||
}
|
||||
}
|
||||
if (upperBoundTimesToMetadata.isEmpty()) {
|
||||
if (upperBoundTimesToBlobInfo.isEmpty()) {
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
|
@ -129,23 +124,23 @@ class GcsDiffFileLister {
|
|||
// metadata of a file is sufficient to identify the preceding file, so if we start from the
|
||||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
|
||||
TreeMap<DateTime, BlobInfo> sequence = new TreeMap<>();
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet =
|
||||
!constructDiffSequence(
|
||||
gcsBucket, upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
|
||||
gcsBucket, upperBoundTimesToBlobInfo, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
// Verify that all of the elements in the original set are represented in the sequence. If we
|
||||
// find anything that's not represented, construct a sequence for it.
|
||||
boolean checkForMoreExtraDiffs = true; // Always loop at least once.
|
||||
while (checkForMoreExtraDiffs) {
|
||||
checkForMoreExtraDiffs = false;
|
||||
for (DateTime key : upperBoundTimesToMetadata.descendingKeySet()) {
|
||||
for (DateTime key : upperBoundTimesToBlobInfo.descendingKeySet()) {
|
||||
if (!isInRange(key, fromTime, toTime)) {
|
||||
break;
|
||||
}
|
||||
if (!sequence.containsKey(key)) {
|
||||
constructDiffSequence(gcsBucket, upperBoundTimesToMetadata, fromTime, key, sequence);
|
||||
constructDiffSequence(gcsBucket, upperBoundTimesToBlobInfo, fromTime, key, sequence);
|
||||
checkForMoreExtraDiffs = true;
|
||||
inconsistentFileSet = true;
|
||||
break;
|
||||
|
|
@ -173,15 +168,11 @@ class GcsDiffFileLister {
|
|||
return isBeforeOrAt(start, time) && (end == null || isBeforeOrAt(time, end));
|
||||
}
|
||||
|
||||
private DateTime getLowerBoundTime(GcsFileMetadata metadata) {
|
||||
return DateTime.parse(metadata.getOptions().getUserMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
private DateTime getLowerBoundTime(BlobInfo blobInfo) {
|
||||
return DateTime.parse(blobInfo.getMetadata().get(LOWER_BOUND_CHECKPOINT));
|
||||
}
|
||||
|
||||
private GcsFileMetadata getMetadata(String gcsBucket, String filename) {
|
||||
try {
|
||||
return gcsService.getMetadata(new GcsFilename(gcsBucket, filename));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
private BlobInfo getBlobInfo(String gcsBucket, String filename) {
|
||||
return gcsUtils.getBlobInfo(BlobId.of(gcsBucket, filename));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,12 +25,12 @@ import static org.joda.time.Duration.standardHours;
|
|||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
|
|
@ -51,7 +51,6 @@ import google.registry.util.Clock;
|
|||
import google.registry.util.RequestStatusChecker;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.Optional;
|
||||
import javax.inject.Inject;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
|
@ -70,14 +69,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
|||
static final String PATH = "/_dr/task/replayCommitLogsToSql";
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int BLOCK_SIZE =
|
||||
1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
private static final Duration LEASE_LENGTH = standardHours(1);
|
||||
// Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
|
||||
// request timeouts
|
||||
private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject Response response;
|
||||
@Inject RequestStatusChecker requestStatusChecker;
|
||||
@Inject GcsDiffFileLister diffLister;
|
||||
|
|
@ -122,13 +120,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
|||
}
|
||||
try {
|
||||
logger.atInfo().log("Beginning replay of commit logs.");
|
||||
ImmutableList<GcsFileMetadata> commitLogFiles = getFilesToReplay();
|
||||
ImmutableList<BlobInfo> commitLogFiles = getFilesToReplay();
|
||||
if (dryRun) {
|
||||
response.setStatus(HttpServletResponse.SC_OK);
|
||||
ImmutableList<String> filenames =
|
||||
commitLogFiles.stream()
|
||||
.limit(10)
|
||||
.map(file -> file.getFilename().getObjectName())
|
||||
.map(file -> file.getName())
|
||||
.collect(toImmutableList());
|
||||
String dryRunMessage =
|
||||
"Running in dry-run mode; would have processed %d files. They are (limit 10):\n"
|
||||
|
|
@ -152,22 +150,22 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
|||
}
|
||||
}
|
||||
|
||||
private ImmutableList<GcsFileMetadata> getFilesToReplay() {
|
||||
private ImmutableList<BlobInfo> getFilesToReplay() {
|
||||
// Start at the first millisecond we haven't seen yet
|
||||
DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
|
||||
logger.atInfo().log("Starting replay from: %s.", fromTime);
|
||||
// If there's an inconsistent file set, this will throw IllegalStateException and the job
|
||||
// will try later -- this is likely because an export hasn't finished yet.
|
||||
ImmutableList<GcsFileMetadata> commitLogFiles =
|
||||
ImmutableList<BlobInfo> commitLogFiles =
|
||||
diffLister.listDiffFiles(gcsBucket, fromTime, /* current time */ null);
|
||||
logger.atInfo().log("Found %d new commit log files to process.", commitLogFiles.size());
|
||||
return commitLogFiles;
|
||||
}
|
||||
|
||||
private void replayFiles(ImmutableList<GcsFileMetadata> commitLogFiles) {
|
||||
private void replayFiles(ImmutableList<BlobInfo> commitLogFiles) {
|
||||
DateTime replayTimeoutTime = clock.nowUtc().plus(REPLAY_TIMEOUT_DURATION);
|
||||
int processedFiles = 0;
|
||||
for (GcsFileMetadata metadata : commitLogFiles) {
|
||||
for (BlobInfo metadata : commitLogFiles) {
|
||||
// One transaction per GCS file
|
||||
jpaTm().transact(() -> processFile(metadata));
|
||||
processedFiles++;
|
||||
|
|
@ -181,31 +179,26 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
|
|||
logger.atInfo().log("Replayed %d commit log files to SQL successfully.", processedFiles);
|
||||
}
|
||||
|
||||
private void processFile(GcsFileMetadata metadata) {
|
||||
private void processFile(BlobInfo metadata) {
|
||||
logger.atInfo().log(
|
||||
"Processing commit log file %s of size %d B.",
|
||||
metadata.getFilename(), metadata.getLength());
|
||||
try (InputStream input =
|
||||
Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
"Processing commit log file %s of size %d B.", metadata.getName(), metadata.getSize());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
// Load and process the Datastore transactions one at a time
|
||||
ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
|
||||
CommitLogImports.loadEntitiesByTransaction(input);
|
||||
logger.atInfo().log(
|
||||
"Replaying %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getFilename());
|
||||
allTransactions.size(), metadata.getName());
|
||||
allTransactions.forEach(this::replayTransaction);
|
||||
// if we succeeded, set the last-seen time
|
||||
DateTime checkpoint =
|
||||
DateTime.parse(
|
||||
metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
|
||||
DateTime checkpoint = DateTime.parse(metadata.getName().substring(DIFF_FILE_PREFIX.length()));
|
||||
SqlReplayCheckpoint.set(checkpoint);
|
||||
logger.atInfo().log(
|
||||
"Replayed %d transactions from commit log file %s.",
|
||||
allTransactions.size(), metadata.getFilename());
|
||||
allTransactions.size(), metadata.getName());
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
"Errored out while replaying commit log file " + metadata.getFilename(), e);
|
||||
"Errored out while replaying commit log file " + metadata.getName(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,8 +23,7 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
|||
import com.google.appengine.api.datastore.DatastoreService;
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import com.google.appengine.api.datastore.EntityTranslator;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
|
|
@ -36,6 +35,7 @@ import com.googlecode.objectify.Result;
|
|||
import com.googlecode.objectify.util.ResultNow;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
|
|
@ -48,7 +48,6 @@ import google.registry.request.auth.Auth;
|
|||
import google.registry.util.Retrier;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.channels.Channels;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -69,8 +68,6 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final int BLOCK_SIZE = 1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
public static final String PATH = "/_dr/task/restoreCommitLogs";
|
||||
static final String DRY_RUN_PARAM = "dryRun";
|
||||
static final String FROM_TIME_PARAM = "fromTime";
|
||||
|
|
@ -80,7 +77,8 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
private static final ImmutableSet<RegistryEnvironment> FORBIDDEN_ENVIRONMENTS =
|
||||
ImmutableSet.of(RegistryEnvironment.PRODUCTION, RegistryEnvironment.SANDBOX);
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Parameter(DRY_RUN_PARAM) boolean dryRun;
|
||||
@Inject @Parameter(FROM_TIME_PARAM) DateTime fromTime;
|
||||
@Inject @Parameter(TO_TIME_PARAM) DateTime toTime;
|
||||
|
|
@ -109,17 +107,16 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
}
|
||||
String gcsBucket = gcsBucketOverride.orElse(defaultGcsBucket);
|
||||
logger.atInfo().log("Restoring from %s.", gcsBucket);
|
||||
List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
|
||||
List<BlobInfo> diffFiles = diffLister.listDiffFiles(gcsBucket, fromTime, toTime);
|
||||
if (diffFiles.isEmpty()) {
|
||||
logger.atInfo().log("Nothing to restore");
|
||||
return;
|
||||
}
|
||||
Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
|
||||
CommitLogCheckpoint lastCheckpoint = null;
|
||||
for (GcsFileMetadata metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
|
||||
try (InputStream input = Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
for (BlobInfo metadata : diffFiles) {
|
||||
logger.atInfo().log("Restoring: %s", metadata.getName());
|
||||
try (InputStream input = gcsUtils.openInputStream(metadata.getBlobId())) {
|
||||
PeekingIterator<ImmutableObject> commitLogs =
|
||||
peekingIterator(createDeserializingIterator(input, true));
|
||||
lastCheckpoint = (CommitLogCheckpoint) commitLogs.next();
|
||||
|
|
|
|||
|
|
@ -416,17 +416,6 @@ public final class RegistryConfig {
|
|||
return Optional.ofNullable(config.cloudDns.servicePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns size of Google Cloud Storage client connection buffer in bytes.
|
||||
*
|
||||
* @see google.registry.gcs.GcsUtils
|
||||
*/
|
||||
@Provides
|
||||
@Config("gcsBufferSize")
|
||||
public static int provideGcsBufferSize() {
|
||||
return 1024 * 1024;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the email address of the admin account on the G Suite app used to perform
|
||||
* administrative actions.
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Verify.verifyNotNull;
|
||||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
|
|
@ -24,11 +23,10 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
|||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Suppliers;
|
||||
|
|
@ -79,9 +77,9 @@ public class ExportDomainListsAction implements Runnable {
|
|||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject DriveConnection driveConnection;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject @Config("domainListsGcsBucket") String gcsBucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject ExportDomainListsAction() {}
|
||||
|
||||
@Override
|
||||
|
|
@ -95,7 +93,7 @@ public class ExportDomainListsAction implements Runnable {
|
|||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(clock.nowUtc(), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsUtils),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
|
|
@ -134,7 +132,7 @@ public class ExportDomainListsAction implements Runnable {
|
|||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
|
|
@ -168,11 +166,9 @@ public class ExportDomainListsAction implements Runnable {
|
|||
}
|
||||
|
||||
protected static boolean exportToGcs(
|
||||
String tld, String domains, String gcsBucket, int gcsBufferSize) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
String tld, String domains, String gcsBucket, GcsUtils gcsUtils) {
|
||||
BlobId blobId = BlobId.of(gcsBucket, tld + ".txt");
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (Throwable e) {
|
||||
|
|
@ -214,7 +210,7 @@ public class ExportDomainListsAction implements Runnable {
|
|||
Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection());
|
||||
|
||||
private final String gcsBucket;
|
||||
private final int gcsBufferSize;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
/**
|
||||
* Non-serializable {@link DriveConnection} that will be created when an instance of {@link
|
||||
|
|
@ -224,9 +220,9 @@ public class ExportDomainListsAction implements Runnable {
|
|||
*/
|
||||
private transient DriveConnection driveConnection;
|
||||
|
||||
public ExportDomainListsReducer(String gcsBucket, int gcsBufferSize) {
|
||||
public ExportDomainListsReducer(String gcsBucket, GcsUtils gcsUtils) {
|
||||
this.gcsBucket = gcsBucket;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
|
|
@ -240,7 +236,7 @@ public class ExportDomainListsAction implements Runnable {
|
|||
ImmutableList<String> domains = ImmutableList.sortedCopyOf(() -> fqdns);
|
||||
String domainsList = Joiner.on('\n').join(domains);
|
||||
logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize)) {
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsUtils)) {
|
||||
getContext().incrementCounter("domain lists successful written out to GCS");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to GCS");
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.gcs;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import dagger.Module;
|
||||
import dagger.Provides;
|
||||
|
||||
/** Dagger module for {@link GcsService}. */
|
||||
@Module
|
||||
public final class GcsServiceModule {
|
||||
|
||||
private static final GcsService gcsService =
|
||||
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
|
||||
|
||||
@Provides
|
||||
static GcsService provideGcsService() {
|
||||
return gcsService;
|
||||
}
|
||||
}
|
||||
|
|
@ -14,30 +14,39 @@
|
|||
|
||||
package google.registry.gcs;
|
||||
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterables.getLast;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.cloud.storage.Blob;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.Storage;
|
||||
import com.google.cloud.storage.Storage.BlobListOption;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.CredentialModule.DefaultCredential;
|
||||
import google.registry.util.GoogleCredentialsBundle;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.io.Serializable;
|
||||
import java.nio.channels.Channels;
|
||||
import javax.annotation.CheckReturnValue;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/** Utilities for working with Google Cloud Storage. */
|
||||
public class GcsUtils {
|
||||
/**
|
||||
* Utilities for working with Google Cloud Storage.
|
||||
*
|
||||
* <p>It is {@link Serializable} so that it can be used in MapReduce or Beam.
|
||||
*/
|
||||
public class GcsUtils implements Serializable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
|
|
@ -48,30 +57,65 @@ public class GcsUtils {
|
|||
.put("json", MediaType.JSON_UTF_8)
|
||||
.build();
|
||||
|
||||
private final GcsService gcsService;
|
||||
private final int bufferSize;
|
||||
private final StorageOptions storageOptions;
|
||||
|
||||
@Inject
|
||||
public GcsUtils(GcsService gcsService, @Config("gcsBufferSize") int bufferSize) {
|
||||
this.gcsService = gcsService;
|
||||
this.bufferSize = bufferSize;
|
||||
private Storage storage() {
|
||||
return storageOptions.getService();
|
||||
}
|
||||
|
||||
/** Opens a GCS file for reading as an {@link InputStream} with prefetching. */
|
||||
@Inject
|
||||
public GcsUtils(@DefaultCredential GoogleCredentialsBundle credentialsBundle) {
|
||||
this(
|
||||
StorageOptions.newBuilder()
|
||||
.setCredentials(credentialsBundle.getGoogleCredentials())
|
||||
.build());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public GcsUtils(StorageOptions storageOptions) {
|
||||
this.storageOptions = storageOptions;
|
||||
}
|
||||
|
||||
/** Opens a GCS file for reading as an {@link InputStream}. */
|
||||
@CheckReturnValue
|
||||
public InputStream openInputStream(GcsFilename filename) {
|
||||
return Channels.newInputStream(gcsService.openPrefetchingReadChannel(filename, 0, bufferSize));
|
||||
public InputStream openInputStream(BlobId blobId) {
|
||||
return Channels.newInputStream(storage().reader(blobId));
|
||||
}
|
||||
|
||||
/** Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files. */
|
||||
@CheckReturnValue
|
||||
public OutputStream openOutputStream(GcsFilename filename) throws IOException {
|
||||
return Channels.newOutputStream(gcsService.createOrReplace(filename, getOptions(filename)));
|
||||
public OutputStream openOutputStream(BlobId blobId) {
|
||||
return Channels.newOutputStream(storage().writer(createBlobInfo(blobId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Opens a GCS file for writing as an {@link OutputStream}, overwriting existing files and setting
|
||||
* the given metadata.
|
||||
*/
|
||||
@CheckReturnValue
|
||||
public OutputStream openOutputStream(BlobId blobId, ImmutableMap<String, String> metadata) {
|
||||
return Channels.newOutputStream(
|
||||
storage().writer(BlobInfo.newBuilder(blobId).setMetadata(metadata).build()));
|
||||
}
|
||||
|
||||
/** Creates a GCS file with the given byte contents, overwriting existing files. */
|
||||
public void createFromBytes(GcsFilename filename, byte[] bytes) throws IOException {
|
||||
gcsService.createOrReplace(filename, getOptions(filename), ByteBuffer.wrap(bytes));
|
||||
public void createFromBytes(BlobId blobId, byte[] bytes) throws StorageException {
|
||||
createFromBytes(createBlobInfo(blobId), bytes);
|
||||
}
|
||||
|
||||
/** Creates a GCS file with the given byte contents and metadata, overwriting existing files. */
|
||||
public void createFromBytes(BlobInfo blobInfo, byte[] bytes) throws StorageException {
|
||||
storage().create(blobInfo, bytes);
|
||||
}
|
||||
|
||||
/** Read the content of the given GCS file and return it in a byte array. */
|
||||
public byte[] readBytesFrom(BlobId blobId) throws StorageException {
|
||||
return storage().readAllBytes(blobId);
|
||||
}
|
||||
|
||||
/** Delete the given GCS file. */
|
||||
public void delete(BlobId blobId) throws StorageException {
|
||||
storage().delete(blobId);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -86,37 +130,55 @@ public class GcsUtils {
|
|||
*/
|
||||
public ImmutableList<String> listFolderObjects(String bucketName, String prefix)
|
||||
throws IOException {
|
||||
ListResult result =
|
||||
gcsService.list(bucketName, new ListOptions.Builder().setPrefix(prefix).build());
|
||||
final ImmutableList.Builder<String> builder = new ImmutableList.Builder<>();
|
||||
result.forEachRemaining(
|
||||
listItem -> {
|
||||
if (!listItem.isDirectory()) {
|
||||
builder.add(listItem.getName().replaceFirst(prefix, ""));
|
||||
}
|
||||
});
|
||||
return builder.build();
|
||||
return Streams.stream(storage().list(bucketName, BlobListOption.prefix(prefix)).iterateAll())
|
||||
.map(blob -> blob.getName().substring(prefix.length()))
|
||||
.collect(toImmutableList());
|
||||
}
|
||||
|
||||
/** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */
|
||||
public boolean existsAndNotEmpty(GcsFilename file) {
|
||||
GcsFileMetadata metadata;
|
||||
public boolean existsAndNotEmpty(BlobId blobId) {
|
||||
try {
|
||||
metadata = gcsService.getMetadata(file);
|
||||
} catch (IOException e) {
|
||||
Blob blob = storage().get(blobId);
|
||||
return blob != null && blob.getSize() > 0;
|
||||
} catch (StorageException e) {
|
||||
logger.atWarning().withCause(e).log("Failed to check if GCS file exists");
|
||||
return false;
|
||||
}
|
||||
return metadata != null && metadata.getLength() > 0;
|
||||
}
|
||||
|
||||
/** Determines most appropriate {@link GcsFileOptions} based on filename extension. */
|
||||
private static GcsFileOptions getOptions(GcsFilename filename) {
|
||||
GcsFileOptions.Builder builder = new GcsFileOptions.Builder().cacheControl("no-cache");
|
||||
MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(filename.getObjectName())));
|
||||
/** Returns the user defined metadata of a GCS file if the file exists, or an empty map. */
|
||||
public ImmutableMap<String, String> getMetadata(BlobId blobId) throws StorageException {
|
||||
Blob blob = storage().get(blobId);
|
||||
return blob == null ? ImmutableMap.of() : ImmutableMap.copyOf(blob.getMetadata());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the {@link BlobInfo} of the given GCS file.
|
||||
*
|
||||
* <p>Note that a {@link Blob} is returned, but on the {@link BlobInfo} part of it is usable.
|
||||
*/
|
||||
public BlobInfo getBlobInfo(BlobId blobId) throws StorageException {
|
||||
return storage().get(blobId);
|
||||
}
|
||||
|
||||
/** Determines most appropriate {@link BlobInfo} based on filename extension. */
|
||||
private static BlobInfo createBlobInfo(BlobId blobId) {
|
||||
BlobInfo.Builder builder = BlobInfo.newBuilder(blobId).setCacheControl("no-cache");
|
||||
MediaType mediaType = EXTENSIONS.get(getLast(Splitter.on('.').split(blobId.getName())));
|
||||
if (mediaType != null) {
|
||||
builder = builder.mimeType(mediaType.type());
|
||||
builder = builder.setContentType(mediaType.toString());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// These two methods are needed to check whether serialization is done correctly in tests.
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
return obj instanceof GcsUtils && ((GcsUtils) obj).storageOptions.equals(storageOptions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return storageOptions.hashCode();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import google.registry.export.datastore.DatastoreAdminModule;
|
|||
import google.registry.export.sheet.SheetsServiceModule;
|
||||
import google.registry.flows.ServerTridProviderModule;
|
||||
import google.registry.flows.custom.CustomLogicFactoryModule;
|
||||
import google.registry.gcs.GcsServiceModule;
|
||||
import google.registry.groups.DirectoryModule;
|
||||
import google.registry.groups.GroupsModule;
|
||||
import google.registry.groups.GroupssettingsModule;
|
||||
|
|
@ -64,7 +63,6 @@ import javax.inject.Singleton;
|
|||
DirectoryModule.class,
|
||||
DummyKeyringModule.class,
|
||||
DriveModule.class,
|
||||
GcsServiceModule.class,
|
||||
GroupsModule.class,
|
||||
GroupssettingsModule.class,
|
||||
JSchModule.class,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import google.registry.config.RegistryConfig.ConfigModule;
|
|||
import google.registry.export.DriveModule;
|
||||
import google.registry.flows.ServerTridProviderModule;
|
||||
import google.registry.flows.custom.CustomLogicFactoryModule;
|
||||
import google.registry.gcs.GcsServiceModule;
|
||||
import google.registry.groups.DirectoryModule;
|
||||
import google.registry.groups.GroupsModule;
|
||||
import google.registry.groups.GroupssettingsModule;
|
||||
|
|
@ -54,7 +53,6 @@ import javax.inject.Singleton;
|
|||
DirectoryModule.class,
|
||||
DummyKeyringModule.class,
|
||||
DriveModule.class,
|
||||
GcsServiceModule.class,
|
||||
GroupsModule.class,
|
||||
GroupssettingsModule.class,
|
||||
Jackson2Module.class,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ package google.registry.rde;
|
|||
import static google.registry.model.rde.RdeMode.THIN;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
|
|
@ -85,10 +85,10 @@ public final class BrdaCopyAction implements Runnable {
|
|||
|
||||
private void copyAsRyde() throws IOException {
|
||||
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
|
||||
GcsFilename xmlFilename = new GcsFilename(stagingBucket, prefix + ".xml.ghostryde");
|
||||
GcsFilename xmlLengthFilename = new GcsFilename(stagingBucket, prefix + ".xml.length");
|
||||
GcsFilename rydeFile = new GcsFilename(brdaBucket, prefix + ".ryde");
|
||||
GcsFilename sigFile = new GcsFilename(brdaBucket, prefix + ".sig");
|
||||
BlobId xmlFilename = BlobId.of(stagingBucket, prefix + ".xml.ghostryde");
|
||||
BlobId xmlLengthFilename = BlobId.of(stagingBucket, prefix + ".xml.length");
|
||||
BlobId rydeFile = BlobId.of(brdaBucket, prefix + ".ryde");
|
||||
BlobId sigFile = BlobId.of(brdaBucket, prefix + ".sig");
|
||||
|
||||
long xmlLength = readXmlLength(xmlLengthFilename);
|
||||
|
||||
|
|
@ -107,7 +107,7 @@ public final class BrdaCopyAction implements Runnable {
|
|||
}
|
||||
|
||||
/** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */
|
||||
private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException {
|
||||
private long readXmlLength(BlobId xmlLengthFilename) throws IOException {
|
||||
try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) {
|
||||
return Ghostryde.readLength(input);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import static google.registry.persistence.transaction.TransactionManagerUtil.tra
|
|||
import static google.registry.request.Action.Method.POST;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
|
|
@ -97,7 +97,7 @@ public final class RdeReportAction implements Runnable, EscrowTask {
|
|||
.orElseThrow(
|
||||
() -> new IllegalStateException("RdeRevision was not set on generated deposit"));
|
||||
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
|
||||
BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde");
|
||||
verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
|
||||
reporter.send(readReportFromGcs(reportFilename));
|
||||
response.setContentType(PLAIN_TEXT_UTF_8);
|
||||
|
|
@ -106,7 +106,7 @@ public final class RdeReportAction implements Runnable, EscrowTask {
|
|||
}
|
||||
|
||||
/** Reads and decrypts the XML file from cloud storage. */
|
||||
private byte[] readReportFromGcs(GcsFilename reportFilename) throws IOException {
|
||||
private byte[] readReportFromGcs(BlobId reportFilename) throws IOException {
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename);
|
||||
InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey)) {
|
||||
return ByteStreams.toByteArray(ghostrydeDecoder);
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableSetMultimap;
|
|||
import com.google.common.collect.Multimaps;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.mapreduce.inputs.NullInput;
|
||||
|
|
@ -206,6 +207,7 @@ public final class RdeStagingAction implements Runnable {
|
|||
@Inject PendingDepositChecker pendingDepositChecker;
|
||||
@Inject RdeStagingReducer.Factory reducerFactory;
|
||||
@Inject Response response;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject @Config("transactionCooldown") Duration transactionCooldown;
|
||||
@Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual;
|
||||
|
|
@ -234,7 +236,7 @@ public final class RdeStagingAction implements Runnable {
|
|||
}
|
||||
ValidationMode validationMode = lenient ? LENIENT : STRICT;
|
||||
RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings);
|
||||
RdeStagingReducer reducer = reducerFactory.create(validationMode);
|
||||
RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils);
|
||||
|
||||
mrRunner
|
||||
.setJobName("Stage escrow deposits for all TLDs")
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ package google.registry.rde;
|
|||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Verify.verify;
|
||||
import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
|
||||
|
|
@ -24,10 +23,9 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
|||
import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
|
|
@ -69,27 +67,27 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
|||
|
||||
private final TaskQueueUtils taskQueueUtils;
|
||||
private final LockHandler lockHandler;
|
||||
private final int gcsBufferSize;
|
||||
private final String bucket;
|
||||
private final Duration lockTimeout;
|
||||
private final byte[] stagingKeyBytes;
|
||||
private final RdeMarshaller marshaller;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
RdeStagingReducer(
|
||||
TaskQueueUtils taskQueueUtils,
|
||||
LockHandler lockHandler,
|
||||
int gcsBufferSize,
|
||||
String bucket,
|
||||
Duration lockTimeout,
|
||||
byte[] stagingKeyBytes,
|
||||
ValidationMode validationMode) {
|
||||
ValidationMode validationMode,
|
||||
GcsUtils gcsUtils) {
|
||||
this.taskQueueUtils = taskQueueUtils;
|
||||
this.lockHandler = lockHandler;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.bucket = bucket;
|
||||
this.lockTimeout = lockTimeout;
|
||||
this.stagingKeyBytes = stagingKeyBytes;
|
||||
this.marshaller = new RdeMarshaller(validationMode);
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -113,8 +111,6 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
|||
|
||||
// Construct things that Dagger would inject if this wasn't serialized.
|
||||
PGPPublicKey stagingKey = PgpHelper.loadPublicKeyBytes(stagingKeyBytes);
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
RdeCounter counter = new RdeCounter();
|
||||
|
||||
// Determine some basic things about the deposit.
|
||||
|
|
@ -130,14 +126,14 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
|||
checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
|
||||
prefix = "manual/" + key.directoryWithTrailingSlash() + prefix;
|
||||
}
|
||||
GcsFilename xmlFilename = new GcsFilename(bucket, prefix + ".xml.ghostryde");
|
||||
BlobId xmlFilename = BlobId.of(bucket, prefix + ".xml.ghostryde");
|
||||
// This file will contain the byte length (ASCII) of the raw unencrypted XML.
|
||||
//
|
||||
// This is necessary because RdeUploadAction creates a tar file which requires that the length
|
||||
// be outputted. We don't want to have to decrypt the entire ghostryde file to determine the
|
||||
// length, so we just save it separately.
|
||||
GcsFilename xmlLengthFilename = new GcsFilename(bucket, prefix + ".xml.length");
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
|
||||
BlobId xmlLengthFilename = BlobId.of(bucket, prefix + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde");
|
||||
|
||||
// These variables will be populated as we write the deposit XML and used for other files.
|
||||
boolean failed = false;
|
||||
|
|
@ -146,8 +142,8 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
|||
// Write a gigantic XML file to GCS. We'll start by opening encrypted out/err file handles.
|
||||
|
||||
logger.atInfo().log("Writing %s and %s", xmlFilename, xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = cloudStorage.openOutputStream(xmlLengthFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(xmlFilename);
|
||||
OutputStream lengthOutput = gcsUtils.openOutputStream(xmlLengthFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey, lengthOutput);
|
||||
Writer output = new OutputStreamWriter(ghostrydeEncoder, UTF_8)) {
|
||||
|
||||
|
|
@ -194,7 +190,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
|||
// This will be sent to ICANN once we're done uploading the big XML to the escrow provider.
|
||||
if (mode == RdeMode.FULL) {
|
||||
logger.atInfo().log("Writing %s", reportFilename);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(reportFilename);
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(reportFilename);
|
||||
OutputStream ghostrydeEncoder = Ghostryde.encoder(gcsOutput, stagingKey)) {
|
||||
counter.makeReport(id, watermark, header, revision).marshal(ghostrydeEncoder, UTF_8);
|
||||
} catch (IOException | XmlException e) {
|
||||
|
|
@ -249,22 +245,21 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
|
|||
static class Factory {
|
||||
@Inject TaskQueueUtils taskQueueUtils;
|
||||
@Inject LockHandler lockHandler;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject @Config("rdeBucket") String bucket;
|
||||
@Inject @Config("rdeStagingLockTimeout") Duration lockTimeout;
|
||||
@Inject @KeyModule.Key("rdeStagingEncryptionKey") byte[] stagingKeyBytes;
|
||||
|
||||
@Inject Factory() {}
|
||||
|
||||
RdeStagingReducer create(ValidationMode validationMode) {
|
||||
RdeStagingReducer create(ValidationMode validationMode, GcsUtils gcsUtils) {
|
||||
return new RdeStagingReducer(
|
||||
taskQueueUtils,
|
||||
lockHandler,
|
||||
gcsBufferSize,
|
||||
bucket,
|
||||
lockTimeout,
|
||||
stagingKeyBytes,
|
||||
validationMode);
|
||||
validationMode,
|
||||
gcsUtils);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
|||
import static java.util.Arrays.asList;
|
||||
|
||||
import com.google.appengine.api.taskqueue.Queue;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.io.ByteStreams;
|
||||
|
|
@ -165,9 +165,9 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
|||
.orElseThrow(
|
||||
() -> new IllegalStateException("RdeRevision was not set on generated deposit"));
|
||||
final String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
|
||||
final GcsFilename xmlFilename = new GcsFilename(bucket, name + ".xml.ghostryde");
|
||||
final GcsFilename xmlLengthFilename = new GcsFilename(bucket, name + ".xml.length");
|
||||
GcsFilename reportFilename = new GcsFilename(bucket, name + "-report.xml.ghostryde");
|
||||
final BlobId xmlFilename = BlobId.of(bucket, name + ".xml.ghostryde");
|
||||
final BlobId xmlLengthFilename = BlobId.of(bucket, name + ".xml.length");
|
||||
BlobId reportFilename = BlobId.of(bucket, name + "-report.xml.ghostryde");
|
||||
verifyFileExists(xmlFilename);
|
||||
verifyFileExists(xmlLengthFilename);
|
||||
verifyFileExists(reportFilename);
|
||||
|
|
@ -187,29 +187,30 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
|||
}
|
||||
|
||||
/**
|
||||
* Performs a blocking upload of a cloud storage XML file to escrow provider, converting
|
||||
* it to the RyDE format along the way by applying tar+compress+encrypt+sign, and saving the
|
||||
* created RyDE file on GCS for future reference.
|
||||
* Performs a blocking upload of a cloud storage XML file to escrow provider, converting it to the
|
||||
* RyDE format along the way by applying tar+compress+encrypt+sign, and saving the created RyDE
|
||||
* file on GCS for future reference.
|
||||
*
|
||||
* <p>This is done by layering a bunch of {@link java.io.FilterOutputStream FilterOutputStreams}
|
||||
* on top of each other in reverse order that turn XML bytes into a RyDE file while
|
||||
* simultaneously uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to
|
||||
* blocking-copy bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline.
|
||||
* on top of each other in reverse order that turn XML bytes into a RyDE file while simultaneously
|
||||
* uploading it to the SFTP endpoint, and then using {@link ByteStreams#copy} to blocking-copy
|
||||
* bytes from the cloud storage {@code InputStream} to the RyDE/SFTP pipeline.
|
||||
*
|
||||
* <p>In pseudo-shell, the whole process looks like the following:
|
||||
*
|
||||
* <pre> {@code
|
||||
* gcs read $xmlFile \ # Get GhostRyDE from cloud storage.
|
||||
* | decrypt | decompress \ # Convert it to XML.
|
||||
* | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
|
||||
* | tee gs://bucket/$rydeFilename.ryde \ # Save a copy of the RyDE file to GCS.
|
||||
* | sftp put $dstUrl/$rydeFilename.ryde \ # Upload to SFTP server.
|
||||
* && sftp put $dstUrl/$rydeFilename.sig </tmp/sig \ # Upload detached signature.
|
||||
* && cat /tmp/sig > gs://bucket/$rydeFilename.sig # Save a copy of signature to GCS.
|
||||
* }</pre>
|
||||
* <pre>{@code
|
||||
* gcs read $xmlFile \ # Get GhostRyDE from cloud storage.
|
||||
* | decrypt | decompress \ # Convert it to XML.
|
||||
* | tar | file | compress | encrypt | sign /tmp/sig \ # Convert it to a RyDE file.
|
||||
* | tee gs://bucket/$rydeFilename.ryde \ # Save a copy of the RyDE file to GCS.
|
||||
* | sftp put $dstUrl/$rydeFilename.ryde \ # Upload to SFTP server.
|
||||
* && sftp put $dstUrl/$rydeFilename.sig </tmp/sig \ # Upload detached signature.
|
||||
* && cat /tmp/sig > gs://bucket/$rydeFilename.sig # Save a copy of signature to GCS.
|
||||
*
|
||||
* }</pre>
|
||||
*/
|
||||
@VisibleForTesting
|
||||
protected void upload(GcsFilename xmlFile, long xmlLength, DateTime watermark, String name)
|
||||
protected void upload(BlobId xmlFile, long xmlLength, DateTime watermark, String name)
|
||||
throws Exception {
|
||||
logger.atInfo().log("Uploading XML file '%s' to remote path '%s'.", xmlFile, uploadUrl);
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile);
|
||||
|
|
@ -218,7 +219,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
|||
JSchSftpChannel ftpChan = session.openSftpChannel()) {
|
||||
ByteArrayOutputStream sigOut = new ByteArrayOutputStream();
|
||||
String rydeFilename = name + ".ryde";
|
||||
GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename);
|
||||
BlobId rydeGcsFilename = BlobId.of(bucket, rydeFilename);
|
||||
try (OutputStream ftpOutput =
|
||||
ftpChan.get().put(rydeFilename, sftpProgressMonitor, OVERWRITE);
|
||||
OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
|
||||
|
|
@ -234,7 +235,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
|||
}
|
||||
String sigFilename = name + ".sig";
|
||||
byte[] signature = sigOut.toByteArray();
|
||||
gcsUtils.createFromBytes(new GcsFilename(bucket, sigFilename), signature);
|
||||
gcsUtils.createFromBytes(BlobId.of(bucket, sigFilename), signature);
|
||||
ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
|
||||
logger.atInfo().log("uploaded %,d bytes: %s", signature.length, sigFilename);
|
||||
}
|
||||
|
|
@ -242,13 +243,13 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
|
|||
}
|
||||
|
||||
/** Reads the contents of a file from Cloud Storage that contains nothing but an integer. */
|
||||
private long readXmlLength(GcsFilename xmlLengthFilename) throws IOException {
|
||||
private long readXmlLength(BlobId xmlLengthFilename) throws IOException {
|
||||
try (InputStream input = gcsUtils.openInputStream(xmlLengthFilename)) {
|
||||
return Ghostryde.readLength(input);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyFileExists(GcsFilename filename) {
|
||||
private void verifyFileExists(BlobId filename) {
|
||||
verify(gcsUtils.existsAndNotEmpty(filename), "Missing file: %s", filename);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ package google.registry.reporting.billing;
|
|||
import static com.google.common.base.Throwables.getRootCause;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.net.MediaType;
|
||||
|
|
@ -72,8 +72,7 @@ public class BillingEmailUtils {
|
|||
void emailOverallInvoice() {
|
||||
try {
|
||||
String invoiceFile = String.format("%s-%s.csv", invoiceFilePrefix, yearMonth);
|
||||
GcsFilename invoiceFilename =
|
||||
new GcsFilename(billingBucket, invoiceDirectoryPrefix + invoiceFile);
|
||||
BlobId invoiceFilename = BlobId.of(billingBucket, invoiceDirectoryPrefix + invoiceFile);
|
||||
try (InputStream in = gcsUtils.openInputStream(invoiceFilename)) {
|
||||
emailService.sendEmail(
|
||||
EmailMessage.newBuilder()
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import static google.registry.request.Action.Method.POST;
|
|||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
|
@ -120,7 +120,7 @@ public final class CopyDetailReportsAction implements Runnable {
|
|||
() -> {
|
||||
try (InputStream input =
|
||||
gcsUtils.openInputStream(
|
||||
new GcsFilename(billingBucket, invoiceDirectoryPrefix + detailReportName))) {
|
||||
BlobId.of(billingBucket, invoiceDirectoryPrefix + detailReportName))) {
|
||||
driveConnection.createOrUpdateFile(
|
||||
detailReportName,
|
||||
MediaType.CSV_UTF_8,
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import static google.registry.reporting.icann.IcannReportingModule.MANIFEST_FILE
|
|||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.ImmutableCollection;
|
||||
|
|
@ -260,7 +260,7 @@ public class IcannReportingStager {
|
|||
Ascii.toLowerCase(reportType.toString()),
|
||||
DateTimeFormat.forPattern("yyyyMM").print(yearMonth));
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, reportFilename);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, reportFilename);
|
||||
gcsUtils.createFromBytes(gcsFilename, reportBytes);
|
||||
logger.atInfo().log("Wrote %d bytes to file location %s", reportBytes.length, gcsFilename);
|
||||
return reportFilename;
|
||||
|
|
@ -269,7 +269,7 @@ public class IcannReportingStager {
|
|||
/** Creates and stores a manifest file on GCS, indicating which reports were generated. */
|
||||
void createAndUploadManifest(String subdir, ImmutableList<String> filenames) throws IOException {
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, subdir);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, MANIFEST_FILE_NAME);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, MANIFEST_FILE_NAME);
|
||||
StringBuilder manifestString = new StringBuilder();
|
||||
filenames.forEach((filename) -> manifestString.append(filename).append("\n"));
|
||||
gcsUtils.createFromBytes(gcsFilename, manifestString.toString().getBytes(UTF_8));
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
|||
import static google.registry.request.Action.Method.POST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Maps;
|
||||
|
|
@ -147,7 +147,7 @@ public final class IcannReportingUploadAction implements Runnable {
|
|||
cursorTimeMinusMonth.getYear(), cursorTimeMinusMonth.getMonthOfYear());
|
||||
String reportBucketname = String.format("%s/%s", reportingBucket, reportSubdir);
|
||||
String filename = getFileName(cursorType, cursorTime, tldStr);
|
||||
final GcsFilename gcsFilename = new GcsFilename(reportBucketname, filename);
|
||||
final BlobId gcsFilename = BlobId.of(reportBucketname, filename);
|
||||
logger.atInfo().log("Reading ICANN report %s from bucket %s", filename, reportBucketname);
|
||||
// Check that the report exists
|
||||
try {
|
||||
|
|
@ -298,18 +298,18 @@ public final class IcannReportingUploadAction implements Runnable {
|
|||
emailService.sendEmail(EmailMessage.create(subject, body, recipient, sender));
|
||||
}
|
||||
|
||||
private byte[] readBytesFromGcs(GcsFilename reportFilename) throws IOException {
|
||||
private byte[] readBytesFromGcs(BlobId reportFilename) throws IOException {
|
||||
try (InputStream gcsInput = gcsUtils.openInputStream(reportFilename)) {
|
||||
return ByteStreams.toByteArray(gcsInput);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyFileExists(GcsFilename gcsFilename) {
|
||||
private void verifyFileExists(BlobId gcsFilename) {
|
||||
checkArgument(
|
||||
gcsUtils.existsAndNotEmpty(gcsFilename),
|
||||
"Object %s in bucket %s not found",
|
||||
gcsFilename.getObjectName(),
|
||||
gcsFilename.getBucketName());
|
||||
gcsFilename.getName(),
|
||||
gcsFilename.getBucket());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ package google.registry.reporting.spec11;
|
|||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
|
@ -62,7 +62,7 @@ public class Spec11RegistrarThreatMatchesParser {
|
|||
}
|
||||
|
||||
/** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */
|
||||
public ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
|
||||
public ImmutableSet<RegistrarThreatMatches> getFromFile(BlobId spec11ReportFilename)
|
||||
throws IOException {
|
||||
if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) {
|
||||
return ImmutableSet.of();
|
||||
|
|
@ -81,7 +81,7 @@ public class Spec11RegistrarThreatMatchesParser {
|
|||
|
||||
public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
|
||||
LocalDate yesterday = date.minusDays(1);
|
||||
GcsFilename gcsFilename = getGcsFilename(yesterday);
|
||||
BlobId gcsFilename = getGcsFilename(yesterday);
|
||||
if (gcsUtils.existsAndNotEmpty(gcsFilename)) {
|
||||
return Optional.of(yesterday);
|
||||
}
|
||||
|
|
@ -98,8 +98,8 @@ public class Spec11RegistrarThreatMatchesParser {
|
|||
return Optional.empty();
|
||||
}
|
||||
|
||||
private GcsFilename getGcsFilename(LocalDate localDate) {
|
||||
return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
|
||||
private BlobId getGcsFilename(LocalDate localDate) {
|
||||
return BlobId.of(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
|
||||
}
|
||||
|
||||
private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
|
||||
package google.registry.tools.server;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterators.filter;
|
||||
import static com.google.common.io.BaseEncoding.base16;
|
||||
|
|
@ -25,11 +24,10 @@ import static google.registry.request.Action.Method.POST;
|
|||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.appengine.tools.mapreduce.Reducer;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
|
@ -98,13 +96,14 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
|
|||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject JsonActionRunner jsonActionRunner;
|
||||
@Inject @Config("zoneFilesBucket") String bucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject @Config("commitLogDatastoreRetention") Duration datastoreRetention;
|
||||
@Inject @Config("dnsDefaultATtl") Duration dnsDefaultATtl;
|
||||
@SuppressWarnings("DurationVariableWithUnits") // false-positive Error Prone check
|
||||
@Inject @Config("dnsDefaultNsTtl") Duration dnsDefaultNsTtl;
|
||||
@Inject @Config("dnsDefaultDsTtl") Duration dnsDefaultDsTtl;
|
||||
@Inject Clock clock;
|
||||
@Inject GcsUtils gcsUtils;
|
||||
|
||||
@Inject GenerateZoneFilesAction() {}
|
||||
|
||||
@Override
|
||||
|
|
@ -140,7 +139,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
|
|||
.runMapreduce(
|
||||
new GenerateBindFileMapper(
|
||||
tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl),
|
||||
new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize),
|
||||
new GenerateBindFileReducer(bucket, exportTime, gcsUtils),
|
||||
ImmutableList.of(new NullInput<>(), createEntityInput(DomainBase.class)))
|
||||
.getLinkToMapreduceConsole();
|
||||
ImmutableList<String> filenames =
|
||||
|
|
@ -236,22 +235,19 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
|
|||
|
||||
private final String bucket;
|
||||
private final DateTime exportTime;
|
||||
private final int gcsBufferSize;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
||||
GenerateBindFileReducer(String bucket, DateTime exportTime, int gcsBufferSize) {
|
||||
GenerateBindFileReducer(String bucket, DateTime exportTime, GcsUtils gcsUtils) {
|
||||
this.bucket = bucket;
|
||||
this.exportTime = exportTime;
|
||||
this.gcsBufferSize = gcsBufferSize;
|
||||
this.gcsUtils = gcsUtils;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(String tld, ReducerInput<String> stanzas) {
|
||||
String stanzaCounter = tld + " stanzas";
|
||||
GcsFilename filename =
|
||||
new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
BlobId filename = BlobId.of(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
|
||||
try (OutputStream gcsOutput = gcsUtils.openOutputStream(filename);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8);
|
||||
PrintWriter writer = new PrintWriter(osWriter)) {
|
||||
writer.printf(HEADER_FORMAT, tld);
|
||||
|
|
|
|||
|
|
@ -24,18 +24,17 @@ import static google.registry.testing.DatabaseHelper.persistResource;
|
|||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.GcsTestingUtils;
|
||||
import google.registry.testing.TestObject;
|
||||
import java.util.List;
|
||||
import org.joda.time.DateTime;
|
||||
|
|
@ -53,8 +52,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
.withOfyTestEntities(TestObject.class)
|
||||
.build();
|
||||
|
||||
/** Local GCS service available for testing. */
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private final DateTime now = DateTime.now(UTC);
|
||||
private final DateTime oneMinuteAgo = now.minusMinutes(1);
|
||||
|
|
@ -63,7 +61,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
task.gcsService = gcsService;
|
||||
task.gcsUtils = gcsUtils;
|
||||
task.gcsBucket = "gcs bucket";
|
||||
task.batchSize = 5;
|
||||
}
|
||||
|
|
@ -84,10 +82,11 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
|
|
@ -95,8 +94,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"0");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported).containsExactly(upperCheckpoint);
|
||||
}
|
||||
|
||||
|
|
@ -139,10 +137,11 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
|
|
@ -150,8 +149,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"4");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in time order, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(2, now.minusDays(1).minusMillis(1));
|
||||
|
|
@ -191,10 +189,11 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
|
|
@ -202,8 +201,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"4");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo);
|
||||
|
|
@ -246,10 +244,11 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
|
|
@ -257,8 +256,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"6");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, oneMinuteAgo);
|
||||
|
|
@ -301,10 +299,11 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
|
|
@ -312,8 +311,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"0");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
// We expect no manifests or mutations, only the upper checkpoint.
|
||||
assertThat(exported).containsExactly(upperCheckpoint);
|
||||
}
|
||||
|
|
@ -359,11 +357,11 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename))
|
||||
.isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
oneMinuteAgo.toString(),
|
||||
|
|
@ -371,8 +369,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"6");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in time order, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(3, oneMinuteAgo.minusDays(2));
|
||||
|
|
@ -415,10 +412,11 @@ public class ExportCommitLogDiffActionTest {
|
|||
|
||||
task.run();
|
||||
|
||||
GcsFilename expectedFilename = new GcsFilename("gcs bucket", "commit_diff_until_" + now);
|
||||
BlobId expectedFilename = BlobId.of("gcs bucket", "commit_diff_until_" + now);
|
||||
assertWithMessage("GCS file not found: " + expectedFilename)
|
||||
.that(gcsService.getMetadata(expectedFilename)).isNotNull();
|
||||
assertThat(gcsService.getMetadata(expectedFilename).getOptions().getUserMetadata())
|
||||
.that(gcsUtils.existsAndNotEmpty(expectedFilename))
|
||||
.isTrue();
|
||||
assertThat(gcsUtils.getMetadata(expectedFilename))
|
||||
.containsExactly(
|
||||
LOWER_BOUND_CHECKPOINT,
|
||||
START_OF_TIME.toString(),
|
||||
|
|
@ -426,8 +424,7 @@ public class ExportCommitLogDiffActionTest {
|
|||
now.toString(),
|
||||
NUM_TRANSACTIONS,
|
||||
"3");
|
||||
List<ImmutableObject> exported =
|
||||
deserializeEntities(GcsTestingUtils.readGcsFile(gcsService, expectedFilename));
|
||||
List<ImmutableObject> exported = deserializeEntities(gcsUtils.readBytesFrom(expectedFilename));
|
||||
assertThat(exported.get(0)).isEqualTo(upperCheckpoint);
|
||||
// We expect these manifests, in the order below, with matching mutations.
|
||||
CommitLogManifest manifest1 = createManifest(1, START_OF_TIME.plusMillis(1));
|
||||
|
|
|
|||
|
|
@ -19,29 +19,24 @@ import static com.google.common.truth.Truth.assertThat;
|
|||
import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
|
||||
import static java.lang.reflect.Proxy.newProxyInstance;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.flogger.LoggerConfig;
|
||||
import com.google.common.testing.TestLogHandler;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.InvocationHandler;
|
||||
import java.lang.reflect.Method;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.logging.LogRecord;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
|
|
@ -55,7 +50,7 @@ public class GcsDiffFileListerTest {
|
|||
|
||||
private final DateTime now = DateTime.now(UTC);
|
||||
private final GcsDiffFileLister diffLister = new GcsDiffFileLister();
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final TestLogHandler logHandler = new TestLogHandler();
|
||||
|
||||
@RegisterExtension
|
||||
|
|
@ -64,25 +59,18 @@ public class GcsDiffFileListerTest {
|
|||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
diffLister.gcsService = gcsService;
|
||||
diffLister.gcsUtils = gcsUtils;
|
||||
diffLister.executor = newDirectExecutorService();
|
||||
for (int i = 0; i < 5; i++) {
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(i)),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(i + 1).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(new byte[]{1, 2, 3}));
|
||||
addGcsFile(i, i + 1);
|
||||
}
|
||||
LoggerConfig.getConfig(GcsDiffFileLister.class).addHandler(logHandler);
|
||||
}
|
||||
|
||||
private Iterable<DateTime> extractTimesFromDiffFiles(List<GcsFileMetadata> diffFiles) {
|
||||
private Iterable<DateTime> extractTimesFromDiffFiles(ImmutableList<BlobInfo> diffFiles) {
|
||||
return transform(
|
||||
diffFiles,
|
||||
metadata ->
|
||||
DateTime.parse(
|
||||
metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length())));
|
||||
blobInfo -> DateTime.parse(blobInfo.getName().substring(DIFF_FILE_PREFIX.length())));
|
||||
}
|
||||
|
||||
private Iterable<DateTime> listDiffFiles(DateTime fromTime, DateTime toTime) {
|
||||
|
|
@ -90,12 +78,12 @@ public class GcsDiffFileListerTest {
|
|||
}
|
||||
|
||||
private void addGcsFile(int fileAge, int prevAge) throws IOException {
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(new byte[]{1, 2, 3}));
|
||||
BlobInfo blobInfo =
|
||||
BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + now.minusMinutes(fileAge)))
|
||||
.setMetadata(
|
||||
ImmutableMap.of(LOWER_BOUND_CHECKPOINT, now.minusMinutes(prevAge).toString()))
|
||||
.build();
|
||||
gcsUtils.createFromBytes(blobInfo, new byte[] {1, 2, 3});
|
||||
}
|
||||
|
||||
private void assertLogContains(String message) {
|
||||
|
|
@ -114,38 +102,11 @@ public class GcsDiffFileListerTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
void testList_patchesHoles() {
|
||||
// Fake out the GCS list() method to return only the first and last file.
|
||||
// We can't use Mockito.spy() because GcsService's impl is final.
|
||||
diffLister.gcsService = (GcsService) newProxyInstance(
|
||||
GcsService.class.getClassLoader(),
|
||||
new Class<?>[] {GcsService.class},
|
||||
new InvocationHandler() {
|
||||
@Override
|
||||
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
|
||||
if (method.getName().equals("list")) {
|
||||
// ListResult is an incredibly annoying thing to construct. It needs to be fed from a
|
||||
// Callable that returns Iterators, each representing a batch of results.
|
||||
return new ListResult(new Callable<Iterator<ListItem>>() {
|
||||
boolean called = false;
|
||||
|
||||
@Override
|
||||
public Iterator<ListItem> call() {
|
||||
try {
|
||||
return called ? null : Iterators.forArray(
|
||||
new ListItem.Builder()
|
||||
.setName(DIFF_FILE_PREFIX + now)
|
||||
.build(),
|
||||
new ListItem.Builder()
|
||||
.setName(DIFF_FILE_PREFIX + now.minusMinutes(4))
|
||||
.build());
|
||||
} finally {
|
||||
called = true;
|
||||
}
|
||||
}});
|
||||
}
|
||||
return method.invoke(gcsService, args);
|
||||
}});
|
||||
void testList_patchesHoles() throws Exception {
|
||||
GcsUtils mockGcsUtils = mock(GcsUtils.class);
|
||||
diffLister.gcsUtils = spy(gcsUtils);
|
||||
when(mockGcsUtils.listFolderObjects(anyString(), anyString()))
|
||||
.thenReturn(ImmutableList.of(now.toString(), now.minusMinutes(4).toString()));
|
||||
DateTime fromTime = now.minusMinutes(4).minusSeconds(1);
|
||||
// Request all files with checkpoint > fromTime.
|
||||
assertThat(listDiffFiles(fromTime, null))
|
||||
|
|
|
|||
|
|
@ -37,14 +37,14 @@ import static org.mockito.Mockito.spy;
|
|||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.ImmutableSortedMap;
|
||||
import com.google.common.truth.Truth8;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
|
|
@ -110,7 +110,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
.build();
|
||||
|
||||
/** Local GCS service. */
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private final ReplayCommitLogsToSqlAction action = new ReplayCommitLogsToSqlAction();
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
|
|
@ -123,13 +123,13 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
action.gcsService = gcsService;
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.response = response;
|
||||
action.requestStatusChecker = requestStatusChecker;
|
||||
action.clock = fakeClock;
|
||||
action.gcsBucket = "gcs bucket";
|
||||
action.diffLister = new GcsDiffFileLister();
|
||||
action.diffLister.gcsService = gcsService;
|
||||
action.diffLister.gcsUtils = gcsUtils;
|
||||
action.diffLister.executor = newDirectExecutorService();
|
||||
ofyTm()
|
||||
.transact(
|
||||
|
|
@ -165,9 +165,9 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
|
||||
Key<CommitLogManifest> manifest2Key =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -182,7 +182,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
|
||||
CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -200,8 +200,8 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
void testReplay_noManifests() throws Exception {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFile(gcsService, createCheckpoint(now.minusMillis(2)));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
saveDiffFile(gcsUtils, createCheckpoint(now.minusMillis(2)));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1)));
|
||||
runAndAssertSuccess(now.minusMillis(1));
|
||||
assertExpectedIds("previous to keep");
|
||||
|
|
@ -214,10 +214,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
|
|
@ -237,10 +237,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
|
|
@ -258,10 +258,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
jpaTm().insertWithoutBackup(TestObject.create("previous to keep"));
|
||||
jpaTm().insertWithoutBackup(TestObject.create("previous to delete"));
|
||||
});
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -276,10 +276,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> jpaTm().put(TestObject.create("existing", "a")));
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMillis(1)),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
|
||||
|
|
@ -294,10 +294,10 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
void testReplay_deleteMissingEntity() throws Exception {
|
||||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> jpaTm().put(TestObject.create("previous to keep", "a")));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1).minusMillis(1));
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMillis(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -332,7 +332,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
TransactionManagerFactory.setJpaTm(() -> spy);
|
||||
// Save in the commit logs the domain and contact (in that order) and the token deletion
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(toDelete))),
|
||||
|
|
@ -375,7 +375,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
TransactionManagerFactory.setJpaTm(() -> spy);
|
||||
// Save two commits -- the deletion, then the new version of the contact
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1).plusMillis(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(contact))),
|
||||
|
|
@ -415,7 +415,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
() -> {
|
||||
try {
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1), now.minusMinutes(1), ImmutableSet.of()),
|
||||
|
|
@ -444,7 +444,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
// Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
|
||||
Key<ClaimsList> claimsListKey = Key.create(ClaimsList.class, 1L);
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -491,7 +491,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")));
|
||||
|
|
@ -504,7 +504,7 @@ public class ReplayCommitLogsToSqlActionTest {
|
|||
DateTime now = fakeClock.nowUtc();
|
||||
jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
|
||||
package google.registry.backup;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.collect.Iterables.transform;
|
||||
import static com.google.common.collect.Maps.toMap;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
|
@ -28,15 +27,16 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
|
|||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.api.datastore.DatastoreServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.io.Resources;
|
||||
import com.google.common.primitives.Longs;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
|
|
@ -51,7 +51,6 @@ import google.registry.testing.TestObject;
|
|||
import google.registry.util.Retrier;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
|
@ -68,7 +67,7 @@ public class RestoreCommitLogsActionTest {
|
|||
|
||||
private final DateTime now = DateTime.now(UTC);
|
||||
private final RestoreCommitLogsAction action = new RestoreCommitLogsAction();
|
||||
private final GcsService gcsService = createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
@RegisterExtension
|
||||
public final AppEngineExtension appEngine =
|
||||
|
|
@ -79,7 +78,7 @@ public class RestoreCommitLogsActionTest {
|
|||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
action.gcsService = gcsService;
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.dryRun = false;
|
||||
action.datastoreService = DatastoreServiceFactory.getDatastoreService();
|
||||
action.fromTime = now.minusMillis(1);
|
||||
|
|
@ -87,7 +86,7 @@ public class RestoreCommitLogsActionTest {
|
|||
action.defaultGcsBucket = GCS_BUCKET;
|
||||
action.gcsBucketOverride = Optional.empty();
|
||||
action.diffLister = new GcsDiffFileLister();
|
||||
action.diffLister.gcsService = gcsService;
|
||||
action.diffLister.gcsUtils = gcsUtils;
|
||||
action.diffLister.executor = newDirectExecutorService();
|
||||
}
|
||||
|
||||
|
|
@ -109,10 +108,10 @@ public class RestoreCommitLogsActionTest {
|
|||
CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
|
||||
Key<CommitLogManifest> manifest2Key =
|
||||
CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(2));
|
||||
Iterable<ImmutableObject> file1CommitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now.minusMinutes(1)),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -128,7 +127,7 @@ public class RestoreCommitLogsActionTest {
|
|||
CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
|
||||
Iterable<ImmutableObject> file2CommitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -149,8 +148,8 @@ public class RestoreCommitLogsActionTest {
|
|||
@Test
|
||||
void testRestore_noManifests() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(gcsService, createCheckpoint(now));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs = saveDiffFile(gcsUtils, createCheckpoint(now));
|
||||
action.run();
|
||||
auditedOfy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
|
|
@ -164,10 +163,10 @@ public class RestoreCommitLogsActionTest {
|
|||
auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
|
||||
Key<CommitLogBucket> bucketKey = getBucketKey(1);
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(bucketKey, now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("a")),
|
||||
|
|
@ -186,10 +185,10 @@ public class RestoreCommitLogsActionTest {
|
|||
.saveWithoutBackup()
|
||||
.entities(TestObject.create("previous to keep"), TestObject.create("previous to delete"))
|
||||
.now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -207,12 +206,10 @@ public class RestoreCommitLogsActionTest {
|
|||
@Test
|
||||
void testRestore_manifestWithNoMutationsOrDeletions() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entities(TestObject.create("previous to keep")).now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null));
|
||||
gcsUtils, createCheckpoint(now), CommitLogManifest.create(getBucketKey(1), now, null));
|
||||
action.run();
|
||||
auditedOfy().clearSessionCache();
|
||||
assertExpectedIds("previous to keep");
|
||||
|
|
@ -225,10 +222,10 @@ public class RestoreCommitLogsActionTest {
|
|||
void testRestore_mutateExistingEntity() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now();
|
||||
Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
|
||||
|
|
@ -245,10 +242,10 @@ public class RestoreCommitLogsActionTest {
|
|||
@Test
|
||||
void testRestore_deleteMissingEntity() throws Exception {
|
||||
auditedOfy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now();
|
||||
saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
|
||||
saveDiffFileNotToRestore(gcsUtils, now.minusMinutes(1));
|
||||
Iterable<ImmutableObject> commitLogs =
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(
|
||||
getBucketKey(1),
|
||||
|
|
@ -270,7 +267,7 @@ public class RestoreCommitLogsActionTest {
|
|||
// imported, in particular, the domain's 'registrant' key can be used by Objectify to load the
|
||||
// contact.
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
Resources.toByteArray(Resources.getResource("google/registry/backup/commitlog.data")),
|
||||
now);
|
||||
action.run();
|
||||
|
|
@ -286,18 +283,18 @@ public class RestoreCommitLogsActionTest {
|
|||
return CommitLogCheckpoint.create(now, toMap(getBucketIds(), x -> now));
|
||||
}
|
||||
|
||||
static void saveDiffFile(GcsService gcsService, byte[] rawBytes, DateTime timestamp)
|
||||
static void saveDiffFile(GcsUtils gcsUtils, byte[] rawBytes, DateTime timestamp)
|
||||
throws IOException {
|
||||
gcsService.createOrReplace(
|
||||
new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp),
|
||||
new GcsFileOptions.Builder()
|
||||
.addUserMetadata(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString())
|
||||
.build(),
|
||||
ByteBuffer.wrap(rawBytes));
|
||||
BlobInfo blobInfo =
|
||||
BlobInfo.newBuilder(BlobId.of(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp))
|
||||
.setMetadata(
|
||||
ImmutableMap.of(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString()))
|
||||
.build();
|
||||
gcsUtils.createFromBytes(blobInfo, rawBytes);
|
||||
}
|
||||
|
||||
static Iterable<ImmutableObject> saveDiffFile(
|
||||
GcsService gcsService, CommitLogCheckpoint checkpoint, ImmutableObject... entities)
|
||||
GcsUtils gcsUtils, CommitLogCheckpoint checkpoint, ImmutableObject... entities)
|
||||
throws IOException {
|
||||
DateTime now = checkpoint.getCheckpointTime();
|
||||
List<ImmutableObject> allEntities = Lists.asList(checkpoint, entities);
|
||||
|
|
@ -305,13 +302,13 @@ public class RestoreCommitLogsActionTest {
|
|||
for (ImmutableObject entity : allEntities) {
|
||||
serializeEntity(entity, output);
|
||||
}
|
||||
saveDiffFile(gcsService, output.toByteArray(), now);
|
||||
saveDiffFile(gcsUtils, output.toByteArray(), now);
|
||||
return allEntities;
|
||||
}
|
||||
|
||||
static void saveDiffFileNotToRestore(GcsService gcsService, DateTime now) throws Exception {
|
||||
static void saveDiffFileNotToRestore(GcsUtils gcsUtils, DateTime now) throws Exception {
|
||||
saveDiffFile(
|
||||
gcsService,
|
||||
gcsUtils,
|
||||
createCheckpoint(now),
|
||||
CommitLogManifest.create(getBucketKey(1), now, null),
|
||||
CommitLogMutation.create(
|
||||
|
|
|
|||
|
|
@ -14,14 +14,12 @@
|
|||
|
||||
package google.registry.export;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_FILENAME;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
|
|
@ -29,12 +27,13 @@ import static org.mockito.Mockito.mock;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.export.ExportDomainListsAction.ExportDomainListsReducer;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
|
|
@ -46,7 +45,6 @@ import google.registry.testing.InjectExtension;
|
|||
import google.registry.testing.TestOfyAndSql;
|
||||
import google.registry.testing.TestOfyOnly;
|
||||
import google.registry.testing.mapreduce.MapreduceTestCase;
|
||||
import java.io.FileNotFoundException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Order;
|
||||
|
|
@ -57,7 +55,7 @@ import org.mockito.ArgumentCaptor;
|
|||
@DualDatabaseTest
|
||||
class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAction> {
|
||||
|
||||
private GcsService gcsService;
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private DriveConnection driveConnection = mock(DriveConnection.class);
|
||||
private ArgumentCaptor<byte[]> bytesExportedToDrive = ArgumentCaptor.forClass(byte[].class);
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
|
|
@ -81,10 +79,9 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
action.mrRunner = makeDefaultRunner();
|
||||
action.response = response;
|
||||
action.gcsBucket = "outputbucket";
|
||||
action.gcsBufferSize = 500;
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.clock = clock;
|
||||
action.driveConnection = driveConnection;
|
||||
gcsService = createGcsService();
|
||||
}
|
||||
|
||||
private void runAction() throws Exception {
|
||||
|
|
@ -117,8 +114,8 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
persistActiveDomain("rudnitzky.tld");
|
||||
persistDeletedDomain("mortuary.tld", DateTime.parse("2001-03-14T10:11:12Z"));
|
||||
runAction();
|
||||
GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8);
|
||||
BlobId existingFile = BlobId.of("outputbucket", "tld.txt");
|
||||
String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8);
|
||||
// Check that it only contains the active domains, not the dead one.
|
||||
assertThat(tlds).isEqualTo("onetwo.tld\nrudnitzky.tld");
|
||||
verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld");
|
||||
|
|
@ -131,17 +128,15 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
persistActiveDomain("rudnitzky.tld");
|
||||
persistActiveDomain("wontgo.testtld");
|
||||
runAction();
|
||||
GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8).trim();
|
||||
BlobId existingFile = BlobId.of("outputbucket", "tld.txt");
|
||||
String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8).trim();
|
||||
// Check that it only contains the domains on the real TLD, and not the test one.
|
||||
assertThat(tlds).isEqualTo("onetwo.tld\nrudnitzky.tld");
|
||||
// Make sure that the test TLD file wasn't written out.
|
||||
GcsFilename nonexistentFile = new GcsFilename("outputbucket", "testtld.txt");
|
||||
assertThrows(FileNotFoundException.class, () -> readGcsFile(gcsService, nonexistentFile));
|
||||
ListResult ls = gcsService.list("outputbucket", ListOptions.DEFAULT);
|
||||
assertThat(ls.next().getName()).isEqualTo("tld.txt");
|
||||
// Make sure that no other files were written out.
|
||||
assertThat(ls.hasNext()).isFalse();
|
||||
BlobId nonexistentFile = BlobId.of("outputbucket", "testtld.txt");
|
||||
assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile));
|
||||
ImmutableList<String> ls = gcsUtils.listFolderObjects("outputbucket", "");
|
||||
assertThat(ls).containsExactly("tld.txt");
|
||||
verifyExportedToDrive("brouhaha", "onetwo.tld\nrudnitzky.tld");
|
||||
verifyNoMoreInteractions(driveConnection);
|
||||
}
|
||||
|
|
@ -160,14 +155,14 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
persistActiveDomain("buddy.tldtwo");
|
||||
persistActiveDomain("cupid.tldthree");
|
||||
runAction();
|
||||
GcsFilename firstTldFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, firstTldFile), UTF_8).trim();
|
||||
BlobId firstTldFile = BlobId.of("outputbucket", "tld.txt");
|
||||
String tlds = new String(gcsUtils.readBytesFrom(firstTldFile), UTF_8).trim();
|
||||
assertThat(tlds).isEqualTo("dasher.tld\nprancer.tld");
|
||||
GcsFilename secondTldFile = new GcsFilename("outputbucket", "tldtwo.txt");
|
||||
String moreTlds = new String(readGcsFile(gcsService, secondTldFile), UTF_8).trim();
|
||||
BlobId secondTldFile = BlobId.of("outputbucket", "tldtwo.txt");
|
||||
String moreTlds = new String(gcsUtils.readBytesFrom(secondTldFile), UTF_8).trim();
|
||||
assertThat(moreTlds).isEqualTo("buddy.tldtwo\nrudolph.tldtwo\nsanta.tldtwo");
|
||||
GcsFilename thirdTldFile = new GcsFilename("outputbucket", "tldthree.txt");
|
||||
String evenMoreTlds = new String(readGcsFile(gcsService, thirdTldFile), UTF_8).trim();
|
||||
BlobId thirdTldFile = BlobId.of("outputbucket", "tldthree.txt");
|
||||
String evenMoreTlds = new String(gcsUtils.readBytesFrom(thirdTldFile), UTF_8).trim();
|
||||
assertThat(evenMoreTlds).isEqualTo("cupid.tldthree");
|
||||
verifyExportedToDrive("brouhaha", "dasher.tld\nprancer.tld");
|
||||
verifyExportedToDrive("hooray", "buddy.tldtwo\nrudolph.tldtwo\nsanta.tldtwo");
|
||||
|
|
|
|||
128
core/src/test/java/google/registry/gcs/GcsUtilsTest.java
Normal file
128
core/src/test/java/google/registry/gcs/GcsUtilsTest.java
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.gcs;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.io.OutputStream;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Unit tests for {@link GcsUtilsTest}. */
|
||||
class GcsUtilsTest {
|
||||
|
||||
private GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private String bucket = "my-bucket";
|
||||
private String filename = "my-file";
|
||||
private BlobId blobId = BlobId.of(bucket, filename);
|
||||
private ImmutableMap<String, String> metadata = ImmutableMap.of("key1", "val1", "Key2", "val2");
|
||||
private final byte[] bytes = new byte[] {'a', 'b', 'c'};
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {}
|
||||
|
||||
@Test
|
||||
void testSerialization_testStorage() throws Exception {
|
||||
assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSerialization_realStorage() throws Exception {
|
||||
gcsUtils = new GcsUtils(StorageOptions.getDefaultInstance());
|
||||
assertThat(deserialize(serialize(gcsUtils))).isEqualTo(gcsUtils);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testStreams() throws Exception {
|
||||
try (OutputStream os = gcsUtils.openOutputStream(blobId, metadata)) {
|
||||
os.write(bytes);
|
||||
os.flush();
|
||||
}
|
||||
assertThat(ByteStreams.toByteArray(gcsUtils.openInputStream(blobId))).isEqualTo(bytes);
|
||||
assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).containsExactlyEntriesIn(metadata);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCreateListReadDelete() throws Exception {
|
||||
gcsUtils.createFromBytes(BlobInfo.newBuilder(blobId).setMetadata(metadata).build(), bytes);
|
||||
assertThat(gcsUtils.existsAndNotEmpty(blobId)).isTrue();
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, "")).containsExactly("my-file");
|
||||
assertThat(gcsUtils.getBlobInfo(blobId).getMetadata()).isEqualTo(metadata);
|
||||
assertThat(gcsUtils.readBytesFrom(blobId)).isEqualTo(bytes);
|
||||
gcsUtils.delete(blobId);
|
||||
assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse();
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, "")).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSetContentType() {
|
||||
blobId = BlobId.of(bucket, "something.json");
|
||||
gcsUtils.createFromBytes(blobId, bytes);
|
||||
assertThat(gcsUtils.getBlobInfo(blobId).getContentType())
|
||||
.isEqualTo(MediaType.JSON_UTF_8.toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testList() throws Exception {
|
||||
ImmutableList<BlobId> blobIds =
|
||||
ImmutableList.of(
|
||||
BlobId.of(bucket, "a/b/xyz.txt"),
|
||||
BlobId.of(bucket, "a/cde.exe"),
|
||||
BlobId.of(bucket, "fgh.jpg"),
|
||||
BlobId.of(bucket, "c/ijk.mp4"));
|
||||
|
||||
for (BlobId blobId : blobIds) {
|
||||
gcsUtils.createFromBytes(blobId, bytes);
|
||||
}
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, ""))
|
||||
.containsExactly("a/b/xyz.txt", "a/cde.exe", "fgh.jpg", "c/ijk.mp4");
|
||||
assertThat(gcsUtils.listFolderObjects(bucket, "a/")).containsExactly("b/xyz.txt", "cde.exe");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testEmptyFile() {
|
||||
gcsUtils.createFromBytes(blobId, new byte[] {});
|
||||
assertThat(gcsUtils.existsAndNotEmpty(blobId)).isFalse();
|
||||
}
|
||||
|
||||
private static byte[] serialize(Object object) throws IOException {
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
ObjectOutputStream oos = new ObjectOutputStream(baos);
|
||||
oos.writeObject(object);
|
||||
oos.flush();
|
||||
return baos.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
private static Object deserialize(byte[] bytes) throws IOException, ClassNotFoundException {
|
||||
try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes))) {
|
||||
return ois.readObject();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,517 @@
|
|||
/*
|
||||
* Copyright 2016 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package google.registry.gcs.backport;
|
||||
|
||||
import com.google.api.client.util.DateTime;
|
||||
import com.google.api.services.storage.model.Bucket;
|
||||
import com.google.api.services.storage.model.ServiceAccount;
|
||||
import com.google.api.services.storage.model.StorageObject;
|
||||
import com.google.cloud.Tuple;
|
||||
import com.google.cloud.storage.Storage;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.cloud.storage.spi.v1.StorageRpc;
|
||||
import com.google.cloud.storage.testing.StorageRpcTestBase;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.file.FileAlreadyExistsException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.annotation.concurrent.NotThreadSafe;
|
||||
|
||||
/**
|
||||
* A bare-bones in-memory implementation of StorageRpc, meant for testing.
|
||||
*
|
||||
* <p>This class is <i>not</i> thread-safe. It's also (currently) limited in the following ways:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Supported
|
||||
* <ul>
|
||||
* <li>object create
|
||||
* <li>object get
|
||||
* <li>object delete
|
||||
* <li>list the contents of a bucket
|
||||
* <li>generations
|
||||
* </ul>
|
||||
* <li>Unsupported
|
||||
* <ul>
|
||||
* <li>bucket create
|
||||
* <li>bucket get
|
||||
* <li>bucket delete
|
||||
* <li>list all buckets
|
||||
* <li>file attributes
|
||||
* <li>patch
|
||||
* <li>continueRewrite
|
||||
* <li>createBatch
|
||||
* <li>checksums, etags
|
||||
* <li>IAM operations
|
||||
* <li>BucketLock operations
|
||||
* <li>HMAC key operations
|
||||
* </ul>
|
||||
* </ul>
|
||||
*/
|
||||
@NotThreadSafe
|
||||
class FakeStorageRpc extends StorageRpcTestBase {
|
||||
|
||||
private static final SimpleDateFormat RFC_3339_FORMATTER =
|
||||
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
|
||||
|
||||
// fullname -> metadata
|
||||
Map<String, StorageObject> metadata = new HashMap<>();
|
||||
// fullname -> contents
|
||||
Map<String, byte[]> contents = new HashMap<>();
|
||||
// fullname -> future contents that will be visible on close.
|
||||
Map<String, byte[]> futureContents = new HashMap<>();
|
||||
|
||||
private final boolean throwIfOption;
|
||||
|
||||
/** @param throwIfOption if true, we throw when given any option */
|
||||
public FakeStorageRpc(boolean throwIfOption) {
|
||||
this.throwIfOption = throwIfOption;
|
||||
}
|
||||
|
||||
// remove all files
|
||||
void reset() {
|
||||
metadata = new HashMap<>();
|
||||
contents = new HashMap<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject create(StorageObject object, InputStream content, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
String key = fullname(object);
|
||||
object.setUpdated(now());
|
||||
metadata.put(key, object);
|
||||
try {
|
||||
contents.put(key, com.google.common.io.ByteStreams.toByteArray(content));
|
||||
} catch (IOException e) {
|
||||
throw new StorageException(e);
|
||||
}
|
||||
// TODO: crc, etc
|
||||
return object;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
String delimiter = null;
|
||||
String preprefix = "";
|
||||
String pageToken = null;
|
||||
long maxResults = Long.MAX_VALUE;
|
||||
for (Map.Entry<Option, ?> e : options.entrySet()) {
|
||||
switch (e.getKey()) {
|
||||
case PAGE_TOKEN:
|
||||
pageToken = (String) e.getValue();
|
||||
break;
|
||||
case PREFIX:
|
||||
preprefix = (String) e.getValue();
|
||||
if (preprefix.startsWith("/")) {
|
||||
preprefix = preprefix.substring(1);
|
||||
}
|
||||
break;
|
||||
case DELIMITER:
|
||||
delimiter = (String) e.getValue();
|
||||
break;
|
||||
case FIELDS:
|
||||
// ignore and return all the fields
|
||||
break;
|
||||
case MAX_RESULTS:
|
||||
maxResults = (Long) e.getValue();
|
||||
break;
|
||||
case USER_PROJECT:
|
||||
// prevent unsupported operation
|
||||
break;
|
||||
default:
|
||||
throw new UnsupportedOperationException("Unknown option: " + e.getKey());
|
||||
}
|
||||
}
|
||||
final String prefix = preprefix;
|
||||
|
||||
List<StorageObject> values = new ArrayList<>();
|
||||
Map<String, StorageObject> folders = new HashMap<>();
|
||||
for (StorageObject so : metadata.values()) {
|
||||
if (!so.getBucket().equals(bucket) || !so.getName().startsWith(prefix)) {
|
||||
continue;
|
||||
}
|
||||
if (processedAsFolder(so, delimiter, prefix, folders)) {
|
||||
continue;
|
||||
}
|
||||
so.setSize(size(so));
|
||||
values.add(so);
|
||||
}
|
||||
values.addAll(folders.values());
|
||||
|
||||
// truncate to max allowed length
|
||||
if (values.size() > maxResults) {
|
||||
List<StorageObject> newValues = new ArrayList<>();
|
||||
for (int i = 0; i < maxResults; i++) {
|
||||
newValues.add(values.get(i));
|
||||
}
|
||||
values = newValues;
|
||||
}
|
||||
|
||||
// null cursor to indicate there is no more data (empty string would cause us to be called
|
||||
// again).
|
||||
// The type cast seems to be necessary to help Java's typesystem remember that collections are
|
||||
// iterable.
|
||||
return Tuple.of(pageToken, (Iterable<StorageObject>) values);
|
||||
}
|
||||
|
||||
/** Returns the requested bucket or {@code null} if not found. */
|
||||
@Override
|
||||
public Bucket get(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Returns the requested storage object or {@code null} if not found. */
|
||||
@Override
|
||||
public StorageObject get(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
// we allow the "ID" option because we need to, but then we give a whole answer anyways
|
||||
// because the caller won't mind the extra fields.
|
||||
if (throwIfOption
|
||||
&& !options.isEmpty()
|
||||
&& options.size() > 1
|
||||
&& options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
String key = fullname(object);
|
||||
if (metadata.containsKey(key)) {
|
||||
StorageObject ret = metadata.get(key);
|
||||
ret.setSize(size(ret));
|
||||
ret.setId(key);
|
||||
|
||||
return ret;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Bucket patch(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject patch(StorageObject storageObject, Map<Option, ?> options)
|
||||
throws StorageException {
|
||||
potentiallyThrow(options);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(Bucket bucket, Map<Option, ?> options) throws StorageException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean delete(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(object);
|
||||
contents.remove(key);
|
||||
return null != metadata.remove(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject compose(
|
||||
Iterable<StorageObject> sources, StorageObject target, Map<Option, ?> targetOptions)
|
||||
throws StorageException {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] load(StorageObject storageObject, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(storageObject);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
return contents.get(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Tuple<String, byte[]> read(
|
||||
StorageObject from, Map<Option, ?> options, long zposition, int zbytes)
|
||||
throws StorageException {
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option op : options.keySet()) {
|
||||
if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) {
|
||||
generationMatch = (Long) options.get(op);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unknown option: " + op);
|
||||
}
|
||||
}
|
||||
String key = fullname(from);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
long position = zposition;
|
||||
int bytes = zbytes;
|
||||
if (position < 0) {
|
||||
position = 0;
|
||||
}
|
||||
byte[] full = contents.get(key);
|
||||
if ((int) position + bytes > full.length) {
|
||||
bytes = full.length - (int) position;
|
||||
}
|
||||
if (bytes <= 0) {
|
||||
// special case: you're trying to read past the end
|
||||
return Tuple.of("etag-goes-here", new byte[0]);
|
||||
}
|
||||
byte[] ret = new byte[bytes];
|
||||
System.arraycopy(full, (int) position, ret, 0, bytes);
|
||||
return Tuple.of("etag-goes-here", ret);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long read(
|
||||
StorageObject from, Map<Option, ?> options, long position, OutputStream outputStream) {
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option op : options.keySet()) {
|
||||
if (op.equals(StorageRpc.Option.IF_GENERATION_MATCH)) {
|
||||
generationMatch = (Long) options.get(op);
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unknown option: " + op);
|
||||
}
|
||||
}
|
||||
String key = fullname(from);
|
||||
if (!contents.containsKey(key)) {
|
||||
throw new StorageException(404, "File not found: " + key);
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
if (position < 0) {
|
||||
position = 0;
|
||||
}
|
||||
byte[] full = contents.get(key);
|
||||
int bytes = (int) (full.length - position);
|
||||
if (bytes <= 0) {
|
||||
// special case: you're trying to read past the end
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
outputStream.write(full, (int) position, bytes);
|
||||
} catch (IOException e) {
|
||||
throw new StorageException(500, "Failed to write to file", e);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String open(StorageObject object, Map<Option, ?> options) throws StorageException {
|
||||
String key = fullname(object);
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option option : options.keySet()) {
|
||||
// this is a bit of a hack, since we don't implement generations.
|
||||
if (option == Option.IF_GENERATION_MATCH) {
|
||||
generationMatch = (Long) options.get(option);
|
||||
}
|
||||
}
|
||||
checkGeneration(key, generationMatch);
|
||||
metadata.put(key, object);
|
||||
|
||||
return fullname(object);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String open(String signedURL) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(
|
||||
String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, int length, boolean last)
|
||||
throws StorageException {
|
||||
writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last);
|
||||
}
|
||||
|
||||
@Override
|
||||
public StorageObject writeWithResponse(
|
||||
String uploadId,
|
||||
byte[] toWrite,
|
||||
int toWriteOffset,
|
||||
long destOffset,
|
||||
int length,
|
||||
boolean last) {
|
||||
// this may have a lot more allocations than ideal, but it'll work.
|
||||
byte[] bytes;
|
||||
if (futureContents.containsKey(uploadId)) {
|
||||
bytes = futureContents.get(uploadId);
|
||||
if (bytes.length < length + destOffset) {
|
||||
byte[] newBytes = new byte[(int) (length + destOffset)];
|
||||
System.arraycopy(bytes, 0, newBytes, (int) 0, bytes.length);
|
||||
bytes = newBytes;
|
||||
}
|
||||
} else {
|
||||
bytes = new byte[(int) (length + destOffset)];
|
||||
}
|
||||
System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length);
|
||||
// we want to mimic the GCS behavior that file contents are only visible on close.
|
||||
StorageObject storageObject = null;
|
||||
if (last) {
|
||||
contents.put(uploadId, bytes);
|
||||
futureContents.remove(uploadId);
|
||||
if (metadata.containsKey(uploadId)) {
|
||||
storageObject = metadata.get(uploadId);
|
||||
storageObject.setUpdated(now());
|
||||
Long generation = storageObject.getGeneration();
|
||||
if (null == generation) {
|
||||
generation = Long.valueOf(0);
|
||||
}
|
||||
storageObject.setGeneration(++generation);
|
||||
metadata.put(uploadId, storageObject);
|
||||
}
|
||||
} else {
|
||||
futureContents.put(uploadId, bytes);
|
||||
}
|
||||
return storageObject;
|
||||
}
|
||||
|
||||
@Override
|
||||
public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException {
|
||||
String sourceKey = fullname(rewriteRequest.source);
|
||||
|
||||
// a little hackish, just good enough for the tests to work.
|
||||
if (!contents.containsKey(sourceKey)) {
|
||||
throw new StorageException(404, "File not found: " + sourceKey);
|
||||
}
|
||||
|
||||
// if non-null, then we check the file's at that generation.
|
||||
Long generationMatch = null;
|
||||
for (Option option : rewriteRequest.targetOptions.keySet()) {
|
||||
// this is a bit of a hack, since we don't implement generations.
|
||||
if (option == Option.IF_GENERATION_MATCH) {
|
||||
generationMatch = (Long) rewriteRequest.targetOptions.get(option);
|
||||
}
|
||||
}
|
||||
|
||||
String destKey = fullname(rewriteRequest.target);
|
||||
|
||||
// if this is a new file, set generation to 1, else increment the existing generation
|
||||
long generation = 1;
|
||||
if (metadata.containsKey(destKey)) {
|
||||
generation = metadata.get(destKey).getGeneration() + 1;
|
||||
}
|
||||
|
||||
checkGeneration(destKey, generationMatch);
|
||||
|
||||
byte[] data = contents.get(sourceKey);
|
||||
|
||||
rewriteRequest.target.setGeneration(generation);
|
||||
rewriteRequest.target.setSize(BigInteger.valueOf(data.length));
|
||||
rewriteRequest.target.setUpdated(metadata.get(sourceKey).getUpdated());
|
||||
|
||||
metadata.put(destKey, rewriteRequest.target);
|
||||
|
||||
contents.put(destKey, Arrays.copyOf(data, data.length));
|
||||
return new RewriteResponse(
|
||||
rewriteRequest,
|
||||
rewriteRequest.target,
|
||||
data.length,
|
||||
true,
|
||||
"rewriteToken goes here",
|
||||
data.length);
|
||||
}
|
||||
|
||||
private static DateTime now() {
|
||||
return DateTime.parseRfc3339(RFC_3339_FORMATTER.format(new Date()));
|
||||
}
|
||||
|
||||
private String fullname(StorageObject so) {
|
||||
return (so.getBucket() + "/" + so.getName());
|
||||
}
|
||||
|
||||
private BigInteger size(StorageObject so) {
|
||||
String key = fullname(so);
|
||||
|
||||
if (contents.containsKey(key)) {
|
||||
return BigInteger.valueOf(contents.get(key).length);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private void potentiallyThrow(Map<Option, ?> options) throws UnsupportedOperationException {
|
||||
if (throwIfOption && !options.isEmpty()) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw if we're asking for generation 0 and the file exists, or if the requested generation
|
||||
* number doesn't match what is asked.
|
||||
*
|
||||
* @param key
|
||||
* @param generationMatch
|
||||
*/
|
||||
private void checkGeneration(String key, Long generationMatch) {
|
||||
if (null == generationMatch) {
|
||||
return;
|
||||
}
|
||||
if (generationMatch == 0 && metadata.containsKey(key)) {
|
||||
throw new StorageException(new FileAlreadyExistsException(key));
|
||||
}
|
||||
if (generationMatch != 0) {
|
||||
Long generation = metadata.get(key).getGeneration();
|
||||
if (!generationMatch.equals(generation)) {
|
||||
throw new StorageException(
|
||||
404, "Generation mismatch. Requested " + generationMatch + " but got " + generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if this is a folder. Adds it to folders if it isn't already there.
|
||||
private static boolean processedAsFolder(
|
||||
StorageObject so,
|
||||
String delimiter,
|
||||
String prefix, /* inout */
|
||||
Map<String, StorageObject> folders) {
|
||||
if (delimiter == null) {
|
||||
return false;
|
||||
}
|
||||
int nextSlash = so.getName().indexOf(delimiter, prefix.length());
|
||||
if (nextSlash < 0) {
|
||||
return false;
|
||||
}
|
||||
String folderName = so.getName().substring(0, nextSlash + 1);
|
||||
if (folders.containsKey(folderName)) {
|
||||
return true;
|
||||
}
|
||||
StorageObject fakeFolder = new StorageObject();
|
||||
fakeFolder.setName(folderName);
|
||||
fakeFolder.setBucket(so.getBucket());
|
||||
fakeFolder.setGeneration(so.getGeneration());
|
||||
fakeFolder.set("isDirectory", true);
|
||||
fakeFolder.setSize(BigInteger.ZERO);
|
||||
folders.put(folderName, fakeFolder);
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceAccount getServiceAccount(String projectId) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
/*
|
||||
* Copyright 2016 Google LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package google.registry.gcs.backport;
|
||||
|
||||
import com.google.cloud.ServiceRpc;
|
||||
import com.google.cloud.spi.ServiceRpcFactory;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
|
||||
/**
|
||||
* Utility to create an in-memory storage configuration for testing. Storage options can be obtained
|
||||
* via the {@link #getOptions()} method. Returned options will point to FakeStorageRpc.
|
||||
*
|
||||
* <p>Note, the created in-memory storage configuration supports limited set of operations and is
|
||||
* <b>not</b> thread-safe:
|
||||
*
|
||||
* <ul>
|
||||
* <li>Supported operations
|
||||
* <ul>
|
||||
* <li>object create
|
||||
* <li>object get
|
||||
* <li>object delete
|
||||
* <li>list the contents of a bucket
|
||||
* </ul>
|
||||
* <li>Unsupported operations
|
||||
* <ul>
|
||||
* <li>bucket create
|
||||
* <li>bucket get
|
||||
* <li>bucket delete
|
||||
* <li>list all buckets
|
||||
* <li>generations
|
||||
* <li>file attributes
|
||||
* <li>patch
|
||||
* <li>continueRewrite
|
||||
* <li>createBatch
|
||||
* <li>checksums, etags
|
||||
* <li>IAM operations
|
||||
* </ul>
|
||||
* </ul>
|
||||
*
|
||||
* {@link FakeStorageRpc#list(String, java.util.Map)} lists all the objects that have been created
|
||||
* rather than the objects in the provided bucket. Since this class does not support creating,
|
||||
* listing and deleting buckets, the parameter bucket here is not actually used and on serves as a
|
||||
* placeholder.
|
||||
*/
|
||||
public final class LocalStorageHelper {
|
||||
|
||||
// used for testing. Will throw if you pass it an option.
|
||||
private static final FakeStorageRpc instance = new FakeStorageRpc(true);
|
||||
|
||||
private LocalStorageHelper() {}
|
||||
|
||||
/**
|
||||
* Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, and resets it
|
||||
* first so you start from a clean slate. That instance will throw if you pass it any option.
|
||||
*/
|
||||
public static StorageOptions getOptions() {
|
||||
instance.reset();
|
||||
return StorageOptions.newBuilder()
|
||||
.setProjectId("fake-project-for-testing")
|
||||
.setServiceRpcFactory(new FakeStorageRpcFactory())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance with the given
|
||||
* option.
|
||||
*/
|
||||
public static StorageOptions customOptions(final boolean throwIfOptions) {
|
||||
return StorageOptions.newBuilder()
|
||||
.setProjectId("fake-project-for-testing")
|
||||
.setServiceRpcFactory(new FakeStorageRpcFactory(new FakeStorageRpc(throwIfOptions)))
|
||||
.build();
|
||||
}
|
||||
|
||||
public static class FakeStorageRpcFactory implements ServiceRpcFactory<StorageOptions> {
|
||||
|
||||
private final FakeStorageRpc instance;
|
||||
|
||||
public FakeStorageRpcFactory() {
|
||||
this(LocalStorageHelper.instance);
|
||||
}
|
||||
|
||||
public FakeStorageRpcFactory(FakeStorageRpc instance) {
|
||||
this.instance = instance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServiceRpc create(StorageOptions storageOptions) {
|
||||
return instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
The files here are directly lifted
|
||||
from [googleapis/java-storage-nio](https://github.com/googleapis/java-storage-nio/tree/master/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/testing)
|
||||
. They are needed because the `StorageOptions` returned from `LocalStorageHelper.getOptions()`
|
||||
should be serializable for the `GcsUtils` class itself to be serializable in tests, but is not. The
|
||||
bug is [fixed](https://github.com/googleapis/java-storage-nio/pull/606) upstream. However, the
|
||||
current released package does not contain the fix yet.
|
||||
|
||||
They are not put under `common/testing` because we do not want to introduce a dependency on the
|
||||
testing configuration from core.
|
||||
|
|
@ -16,23 +16,20 @@ package google.registry.rde;
|
|||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static com.google.common.truth.Truth.assertWithMessage;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.SystemInfo.hasCommand;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assumptions.assumeTrue;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.io.ByteSource;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.BouncyCastleProviderExtension;
|
||||
import google.registry.testing.FakeKeyringModule;
|
||||
import google.registry.testing.GcsTestingUtils;
|
||||
import google.registry.testing.GpgSystemCommandExtension;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
|
@ -52,14 +49,12 @@ public class BrdaCopyActionTest {
|
|||
|
||||
private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml");
|
||||
|
||||
private static final GcsFilename STAGE_FILE =
|
||||
new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde");
|
||||
private static final GcsFilename STAGE_LENGTH_FILE =
|
||||
new GcsFilename("keg", "lol_2010-10-17_thin_S1_R0.xml.length");
|
||||
private static final GcsFilename RYDE_FILE =
|
||||
new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.ryde");
|
||||
private static final GcsFilename SIG_FILE =
|
||||
new GcsFilename("tub", "lol_2010-10-17_thin_S1_R0.sig");
|
||||
private static final BlobId STAGE_FILE =
|
||||
BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.ghostryde");
|
||||
private static final BlobId STAGE_LENGTH_FILE =
|
||||
BlobId.of("keg", "lol_2010-10-17_thin_S1_R0.xml.length");
|
||||
private static final BlobId RYDE_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.ryde");
|
||||
private static final BlobId SIG_FILE = BlobId.of("tub", "lol_2010-10-17_thin_S1_R0.sig");
|
||||
|
||||
@RegisterExtension
|
||||
public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension();
|
||||
|
|
@ -89,8 +84,7 @@ public class BrdaCopyActionTest {
|
|||
}
|
||||
}
|
||||
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final BrdaCopyAction action = new BrdaCopyAction();
|
||||
|
||||
@BeforeEach
|
||||
|
|
@ -105,9 +99,8 @@ public class BrdaCopyActionTest {
|
|||
action.stagingDecryptionKey = decryptKey;
|
||||
|
||||
byte[] xml = DEPOSIT_XML.read();
|
||||
GcsTestingUtils.writeGcsFile(gcsService, STAGE_FILE, Ghostryde.encode(xml, encryptKey));
|
||||
GcsTestingUtils.writeGcsFile(gcsService, STAGE_LENGTH_FILE,
|
||||
Long.toString(xml.length).getBytes(UTF_8));
|
||||
gcsUtils.createFromBytes(STAGE_FILE, Ghostryde.encode(xml, encryptKey));
|
||||
gcsUtils.createFromBytes(STAGE_LENGTH_FILE, Long.toString(xml.length).getBytes(UTF_8));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -124,7 +117,7 @@ public class BrdaCopyActionTest {
|
|||
action.run();
|
||||
|
||||
File rydeTmp = new File(gpg.getCwd(), "ryde");
|
||||
Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp);
|
||||
Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp);
|
||||
Process pid =
|
||||
gpg.exec(
|
||||
"gpg",
|
||||
|
|
@ -172,8 +165,8 @@ public class BrdaCopyActionTest {
|
|||
|
||||
File rydeTmp = new File(gpg.getCwd(), "ryde");
|
||||
File sigTmp = new File(gpg.getCwd(), "ryde.sig");
|
||||
Files.write(readGcsFile(gcsService, RYDE_FILE), rydeTmp);
|
||||
Files.write(readGcsFile(gcsService, SIG_FILE), sigTmp);
|
||||
Files.write(gcsUtils.readBytesFrom(RYDE_FILE), rydeTmp);
|
||||
Files.write(gcsUtils.readBytesFrom(SIG_FILE), sigTmp);
|
||||
|
||||
Process pid = gpg.exec("gpg", "--verify", sigTmp.toString(), rydeTmp.toString());
|
||||
String stderr = slurp(pid.getErrorStream());
|
||||
|
|
|
|||
|
|
@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
|||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.loadByKey;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.deleteGcsFile;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.joda.time.Duration.standardDays;
|
||||
|
|
@ -41,13 +39,12 @@ import com.google.appengine.api.urlfetch.HTTPHeader;
|
|||
import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
import com.google.appengine.api.urlfetch.URLFetchService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteSource;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.rde.RdeRevision;
|
||||
import google.registry.model.registry.Registry;
|
||||
|
|
@ -95,9 +92,9 @@ public class RdeReportActionTest {
|
|||
private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
|
||||
private final HTTPResponse httpResponse = mock(HTTPResponse.class);
|
||||
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsFilename reportFile =
|
||||
new GcsFilename("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final BlobId reportFile =
|
||||
BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
|
||||
|
||||
private RdeReportAction createAction() {
|
||||
RdeReporter reporter = new RdeReporter();
|
||||
|
|
@ -106,7 +103,7 @@ public class RdeReportActionTest {
|
|||
reporter.password = "foo";
|
||||
reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
|
||||
RdeReportAction action = new RdeReportAction();
|
||||
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.response = response;
|
||||
action.bucket = "tub";
|
||||
action.tld = "test";
|
||||
|
|
@ -126,7 +123,7 @@ public class RdeReportActionTest {
|
|||
Cursor.create(RDE_REPORT, DateTime.parse("2006-06-06TZ"), Registry.get("test")));
|
||||
persistResource(
|
||||
Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
|
||||
writeGcsFile(gcsService, reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
|
||||
}
|
||||
|
||||
|
|
@ -169,11 +166,10 @@ public class RdeReportActionTest {
|
|||
|
||||
@TestOfyAndSql
|
||||
void testRunWithLock_regeneratedReport() throws Exception {
|
||||
deleteGcsFile(gcsService, reportFile);
|
||||
GcsFilename newReport =
|
||||
new GcsFilename("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde");
|
||||
gcsUtils.delete(reportFile);
|
||||
BlobId newReport = BlobId.of("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde");
|
||||
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
|
||||
writeGcsFile(gcsService, newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
|
||||
package google.registry.rde;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.common.Cursor.CursorType.BRDA;
|
||||
import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
|
||||
|
|
@ -26,7 +25,6 @@ import static google.registry.rde.RdeFixtures.makeHostResource;
|
|||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.DatabaseHelper.persistResourceWithCommitLog;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.TaskQueueHelper.assertAtLeastOneTaskIsEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
|
|
@ -36,16 +34,13 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
|||
import static java.util.Arrays.asList;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.net.InetAddresses;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
|
|
@ -98,16 +93,16 @@ import org.junit.jupiter.api.extension.RegisterExtension;
|
|||
/** Unit tests for {@link RdeStagingAction}. */
|
||||
public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
||||
|
||||
private static final GcsFilename XML_FILE =
|
||||
new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
private static final GcsFilename LENGTH_FILE =
|
||||
new GcsFilename("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length");
|
||||
private static final BlobId XML_FILE =
|
||||
BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
private static final BlobId LENGTH_FILE =
|
||||
BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.length");
|
||||
|
||||
@RegisterExtension public final InjectExtension inject = new InjectExtension();
|
||||
|
||||
private final FakeClock clock = new FakeClock();
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final List<? super XjcRdeContentType> alreadyExtracted = new ArrayList<>();
|
||||
|
||||
private static PGPPublicKey encryptKey;
|
||||
|
|
@ -131,7 +126,6 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.reducerFactory = new RdeStagingReducer.Factory();
|
||||
action.reducerFactory.taskQueueUtils = new TaskQueueUtils(new Retrier(new SystemSleeper(), 1));
|
||||
action.reducerFactory.lockHandler = new FakeLockHandler(true);
|
||||
action.reducerFactory.gcsBufferSize = 0;
|
||||
action.reducerFactory.bucket = "rde-bucket";
|
||||
action.reducerFactory.lockTimeout = Duration.standardHours(1);
|
||||
action.reducerFactory.stagingKeyBytes = PgpHelper.convertPublicKeyToBytes(encryptKey);
|
||||
|
|
@ -140,6 +134,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.pendingDepositChecker.brdaInterval = Duration.standardDays(7);
|
||||
action.pendingDepositChecker.clock = clock;
|
||||
action.pendingDepositChecker.rdeInterval = Duration.standardDays(1);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.response = response;
|
||||
action.transactionCooldown = Duration.ZERO;
|
||||
action.directory = Optional.empty();
|
||||
|
|
@ -327,7 +322,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey));
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
|
||||
assertThat(header.getTld()).isEqualTo("lol");
|
||||
|
|
@ -358,7 +353,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey));
|
||||
assertThat(deposit.getType()).isEqualTo(XjcRdeDepositTypeType.FULL);
|
||||
assertThat(deposit.getId()).isEqualTo(RdeUtil.timestampToId(DateTime.parse("2000-01-01TZ")));
|
||||
assertThat(deposit.getWatermark()).isEqualTo(DateTime.parse("2000-01-01TZ"));
|
||||
|
|
@ -400,7 +395,7 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey));
|
||||
XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
|
|
@ -494,12 +489,13 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
for (GcsFilename filename : asList(
|
||||
new GcsFilename("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"),
|
||||
new GcsFilename("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) {
|
||||
for (BlobId filename :
|
||||
asList(
|
||||
BlobId.of("rde-bucket", "fop_1971-01-01_full_S1_R0.xml.ghostryde"),
|
||||
BlobId.of("rde-bucket", "fop_1971-01-05_thin_S1_R0.xml.ghostryde"))) {
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey));
|
||||
XjcRdeRegistrar registrar1 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeRegistrar registrar2 = extractAndRemoveContentWithType(XjcRdeRegistrar.class, deposit);
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
|
|
@ -536,10 +532,10 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
GcsFilename filename = new GcsFilename("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
BlobId filename = BlobId.of("rde-bucket", "fop_2000-01-01_full_S1_R0.xml.ghostryde");
|
||||
XjcRdeDeposit deposit =
|
||||
unmarshal(
|
||||
XjcRdeDeposit.class, Ghostryde.decode(readGcsFile(gcsService, filename), decryptKey));
|
||||
XjcRdeDeposit.class, Ghostryde.decode(gcsUtils.readBytesFrom(filename), decryptKey));
|
||||
XjcRdeDomain domain = extractAndRemoveContentWithType(XjcRdeDomain.class, deposit);
|
||||
XjcRdeIdn firstIdn = extractAndRemoveContentWithType(XjcRdeIdn.class, deposit);
|
||||
XjcRdeHeader header = extractAndRemoveContentWithType(XjcRdeHeader.class, deposit);
|
||||
|
|
@ -579,8 +575,8 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
byte[] deposit = Ghostryde.decode(readGcsFile(gcsService, XML_FILE), decryptKey);
|
||||
assertThat(Integer.parseInt(new String(readGcsFile(gcsService, LENGTH_FILE), UTF_8)))
|
||||
byte[] deposit = Ghostryde.decode(gcsUtils.readBytesFrom(XML_FILE), decryptKey);
|
||||
assertThat(Integer.parseInt(new String(gcsUtils.readBytesFrom(LENGTH_FILE), UTF_8)))
|
||||
.isEqualTo(deposit.length);
|
||||
}
|
||||
|
||||
|
|
@ -781,23 +777,20 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce", clock);
|
||||
|
||||
ListResult listResult =
|
||||
gcsService.list("rde-bucket", new ListOptions.Builder().setPrefix("manual/test").build());
|
||||
ImmutableSet<String> filenames =
|
||||
ImmutableList.copyOf(listResult).stream().map(ListItem::getName).collect(toImmutableSet());
|
||||
ImmutableList<String> filenames = gcsUtils.listFolderObjects("rde-bucket", "manual/test/");
|
||||
for (String tld : tlds) {
|
||||
assertThat(filenames)
|
||||
.containsAtLeast(
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_full_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_full_S1_R" + revision + ".xml.length",
|
||||
"manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
"manual/test/" + tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length");
|
||||
tld + "_2000-01-01_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
tld + "_2000-01-01_full_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-01_full_S1_R" + revision + ".xml.length",
|
||||
tld + "_2000-01-01_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-01_thin_S1_R" + revision + ".xml.length",
|
||||
tld + "_2000-01-02_full_S1_R" + revision + "-report.xml.ghostryde",
|
||||
tld + "_2000-01-02_full_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-02_full_S1_R" + revision + ".xml.length",
|
||||
tld + "_2000-01-02_thin_S1_R" + revision + ".xml.ghostryde",
|
||||
tld + "_2000-01-02_thin_S1_R" + revision + ".xml.length");
|
||||
|
||||
assertThat(
|
||||
auditedOfy()
|
||||
|
|
@ -837,8 +830,8 @@ public class RdeStagingActionTest extends MapreduceTestCase<RdeStagingAction> {
|
|||
}
|
||||
|
||||
private String readXml(String objectName) throws IOException, PGPException {
|
||||
GcsFilename file = new GcsFilename("rde-bucket", objectName);
|
||||
return new String(Ghostryde.decode(readGcsFile(gcsService, file), decryptKey), UTF_8);
|
||||
BlobId file = BlobId.of("rde-bucket", objectName);
|
||||
return new String(Ghostryde.decode(gcsUtils.readBytesFrom(file), decryptKey), UTF_8);
|
||||
}
|
||||
|
||||
private <T extends XjcRdeContentType>
|
||||
|
|
|
|||
|
|
@ -20,18 +20,18 @@ import static google.registry.model.rde.RdeMode.FULL;
|
|||
import static google.registry.model.rde.RdeMode.THIN;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static google.registry.util.ResourceUtils.readResourceUtf8;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.mapreduce.ReducerInput;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.StorageException;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.PgpHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
|
|
@ -67,7 +67,7 @@ class RdeStagingReducerTest {
|
|||
AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
|
||||
|
||||
private static final String GCS_BUCKET = "test-rde-bucket";
|
||||
private static final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private static final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private static final PGPPrivateKey decryptionKey =
|
||||
new FakeKeyringModule().get().getRdeStagingDecryptionKey();
|
||||
private static final PGPPublicKey encryptionKey =
|
||||
|
|
@ -95,11 +95,11 @@ class RdeStagingReducerTest {
|
|||
new RdeStagingReducer(
|
||||
new TaskQueueUtils(new Retrier(new FakeSleeper(new FakeClock()), 1)),
|
||||
new FakeLockHandler(true),
|
||||
1024,
|
||||
GCS_BUCKET,
|
||||
Duration.ZERO,
|
||||
PgpHelper.convertPublicKeyToBytes(encryptionKey),
|
||||
ValidationMode.STRICT);
|
||||
ValidationMode.STRICT,
|
||||
gcsUtils);
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
|
|
@ -125,11 +125,10 @@ class RdeStagingReducerTest {
|
|||
compareLength(outputFile, "soy_2000-01-01_thin_S1_R1.xml.length");
|
||||
// BRDA doesn't write a report file.
|
||||
assertThrows(
|
||||
IOException.class,
|
||||
StorageException.class,
|
||||
() ->
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde")));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of(GCS_BUCKET, "soy_2000-01-01_thin_S1_R1-report.xml.ghostryde")));
|
||||
assertThat(loadCursorTime(CursorType.BRDA))
|
||||
.isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
|
||||
assertThat(loadRevision(THIN)).isEqualTo(1);
|
||||
|
|
@ -153,12 +152,10 @@ class RdeStagingReducerTest {
|
|||
compareLength(outputFile, "manual/soy_2000-01-01_thin_S1_R0.xml.length");
|
||||
// BRDA doesn't write a report file.
|
||||
assertThrows(
|
||||
IOException.class,
|
||||
StorageException.class,
|
||||
() ->
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename(
|
||||
GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde")));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of(GCS_BUCKET, "manual/soy_2000-01-01_thin_S1_R0-report.xml.ghostryde")));
|
||||
// No extra operations in manual mode.
|
||||
assertThat(loadCursorTime(CursorType.BRDA)).isEquivalentAccordingToCompareTo(now);
|
||||
assertThat(loadRevision(THIN)).isEqualTo(0);
|
||||
|
|
@ -210,8 +207,7 @@ class RdeStagingReducerTest {
|
|||
private static void compareLength(String outputFile, String lengthFilename) throws IOException {
|
||||
assertThat(String.valueOf(outputFile.getBytes(UTF_8).length))
|
||||
.isEqualTo(
|
||||
new String(
|
||||
readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, lengthFilename)), UTF_8));
|
||||
new String(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, lengthFilename)), UTF_8));
|
||||
}
|
||||
|
||||
private static DateTime loadCursorTime(CursorType type) {
|
||||
|
|
@ -233,8 +229,7 @@ class RdeStagingReducerTest {
|
|||
|
||||
private static String decryptGhostrydeGcsFile(String filename) throws IOException, PGPException {
|
||||
return new String(
|
||||
Ghostryde.decode(
|
||||
readGcsFile(gcsService, new GcsFilename(GCS_BUCKET, filename)), decryptionKey),
|
||||
Ghostryde.decode(gcsUtils.readBytesFrom(BlobId.of(GCS_BUCKET, filename)), decryptionKey),
|
||||
UTF_8);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,8 +24,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
|||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.DatabaseHelper.persistSimpleResource;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static google.registry.testing.SystemInfo.hasCommand;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
|
|
@ -45,9 +43,7 @@ import static org.mockito.Mockito.when;
|
|||
|
||||
import com.google.appengine.api.taskqueue.QueueFactory;
|
||||
import com.google.appengine.api.utils.SystemProperty;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.io.ByteSource;
|
||||
import com.google.common.io.CharStreams;
|
||||
import com.google.common.io.Files;
|
||||
|
|
@ -55,6 +51,7 @@ import com.jcraft.jsch.JSch;
|
|||
import com.jcraft.jsch.JSchException;
|
||||
import com.jcraft.jsch.Session;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.keyring.api.Keyring;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
|
|
@ -94,23 +91,24 @@ import org.mockito.stubbing.OngoingStubbing;
|
|||
@DualDatabaseTest
|
||||
public class RdeUploadActionTest {
|
||||
|
||||
private static final int BUFFER_SIZE = 64 * 1024;
|
||||
private static final ByteSource REPORT_XML = RdeTestData.loadBytes("report.xml");
|
||||
private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml");
|
||||
|
||||
private static final GcsFilename GHOSTRYDE_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde");
|
||||
private static final GcsFilename LENGTH_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0.xml.length");
|
||||
private static final GcsFilename REPORT_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
|
||||
private static final BlobId GHOSTRYDE_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde");
|
||||
private static final BlobId LENGTH_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.length");
|
||||
private static final BlobId REPORT_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
|
||||
|
||||
private static final GcsFilename GHOSTRYDE_R1_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde");
|
||||
private static final GcsFilename LENGTH_R1_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1.xml.length");
|
||||
private static final GcsFilename REPORT_R1_FILE =
|
||||
new GcsFilename("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde");
|
||||
private static final BlobId GHOSTRYDE_R1_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde");
|
||||
private static final BlobId LENGTH_R1_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.length");
|
||||
private static final BlobId REPORT_R1_FILE =
|
||||
BlobId.of("bucket", "tld_2010-10-17_full_S1_R1-report.xml.ghostryde");
|
||||
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
@RegisterExtension final SftpServerExtension sftpd = new SftpServerExtension();
|
||||
|
||||
|
|
@ -139,7 +137,7 @@ public class RdeUploadActionTest {
|
|||
try (Keyring keyring = new FakeKeyringModule().get()) {
|
||||
RdeUploadAction action = new RdeUploadAction();
|
||||
action.clock = clock;
|
||||
action.gcsUtils = new GcsUtils(gcsService, BUFFER_SIZE);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.lazyJsch =
|
||||
() ->
|
||||
JSchModule.provideJSch(
|
||||
|
|
@ -176,22 +174,20 @@ public class RdeUploadActionTest {
|
|||
return jschSpy;
|
||||
}
|
||||
|
||||
private GcsService gcsService;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
// Force "development" mode so we don't try to really connect to GCS.
|
||||
SystemProperty.environment.set(SystemProperty.Environment.Value.Development);
|
||||
gcsService = GcsServiceFactory.createGcsService();
|
||||
|
||||
createTld("tld");
|
||||
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
|
||||
writeGcsFile(gcsService, GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
writeGcsFile(gcsService, GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
writeGcsFile(gcsService, LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
writeGcsFile(gcsService, LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
writeGcsFile(gcsService, REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
writeGcsFile(gcsService, REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(GHOSTRYDE_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(GHOSTRYDE_R1_FILE, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(LENGTH_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
gcsUtils.createFromBytes(LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
|
||||
gcsUtils.createFromBytes(REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
gcsUtils.createFromBytes(REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm()
|
||||
.transact(
|
||||
() -> {
|
||||
|
|
@ -271,9 +267,9 @@ public class RdeUploadActionTest {
|
|||
String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde";
|
||||
String sigFilename = "tld_2010-10-17_full_S1_R0.sig";
|
||||
assertThat(folder.list()).asList().containsExactly(rydeFilename, sigFilename);
|
||||
assertThat(readGcsFile(gcsService, new GcsFilename("bucket", rydeFilename)))
|
||||
assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", rydeFilename)))
|
||||
.isEqualTo(Files.toByteArray(new File(folder, rydeFilename)));
|
||||
assertThat(readGcsFile(gcsService, new GcsFilename("bucket", sigFilename)))
|
||||
assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", sigFilename)))
|
||||
.isEqualTo(Files.toByteArray(new File(folder, sigFilename)));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import static org.mockito.Mockito.times;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
|
|
@ -51,7 +51,7 @@ class BillingEmailUtilsTest {
|
|||
void beforeEach() throws Exception {
|
||||
emailService = mock(SendEmailService.class);
|
||||
gcsUtils = mock(GcsUtils.class);
|
||||
when(gcsUtils.openInputStream(new GcsFilename("test-bucket", "results/REG-INV-2017-10.csv")))
|
||||
when(gcsUtils.openInputStream(BlobId.of("test-bucket", "results/REG-INV-2017-10.csv")))
|
||||
.thenReturn(
|
||||
new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8)));
|
||||
contentCaptor = ArgumentCaptor.forClass(EmailMessage.class);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ package google.registry.reporting.billing;
|
|||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.DatabaseHelper.loadRegistrar;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
|
|
@ -29,11 +28,10 @@ import static org.mockito.Mockito.verifyNoInteractions;
|
|||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
|
|
@ -52,8 +50,7 @@ class CopyDetailReportsActionTest {
|
|||
final AppEngineExtension appEngine =
|
||||
AppEngineExtension.builder().withDatastoreAndCloudSql().build();
|
||||
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
private FakeResponse response;
|
||||
private DriveConnection driveConnection;
|
||||
|
|
@ -80,14 +77,12 @@ class CopyDetailReportsActionTest {
|
|||
|
||||
@Test
|
||||
void testSuccess() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"),
|
||||
"hello,world\n1,2".getBytes(UTF_8));
|
||||
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
|
||||
action.run();
|
||||
|
|
@ -111,14 +106,12 @@ class CopyDetailReportsActionTest {
|
|||
|
||||
@Test
|
||||
void testSuccess_nonDetailReportFiles_notSent() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"),
|
||||
"hello,world\n1,2".getBytes(UTF_8));
|
||||
action.run();
|
||||
verify(driveConnection)
|
||||
|
|
@ -136,9 +129,8 @@ class CopyDetailReportsActionTest {
|
|||
|
||||
@Test
|
||||
void testSuccess_transientIOException_retries() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
when(driveConnection.createOrUpdateFile(any(), any(), any(), any()))
|
||||
.thenThrow(new IOException("expected"))
|
||||
|
|
@ -158,13 +150,12 @@ class CopyDetailReportsActionTest {
|
|||
|
||||
@Test
|
||||
void testFail_tooManyFailures_sendsAlertEmail_continues() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"),
|
||||
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_NewRegistrar_test.csv"),
|
||||
"hello,world\n1,2".getBytes(UTF_8));
|
||||
when(driveConnection.createOrUpdateFile(
|
||||
eq("invoice_details_2017-10_TheRegistrar_hello.csv"), any(), any(), any()))
|
||||
|
|
@ -201,9 +192,8 @@ class CopyDetailReportsActionTest {
|
|||
|
||||
@Test
|
||||
void testFail_registrarDoesntExist_doesntCopy() throws IOException {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
action.run();
|
||||
verifyNoInteractions(driveConnection);
|
||||
|
|
@ -212,9 +202,8 @@ class CopyDetailReportsActionTest {
|
|||
@Test
|
||||
void testFail_noRegistrarFolderId_doesntCopy() throws IOException {
|
||||
persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId(null).build());
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
|
||||
"hola,mundo\n3,4".getBytes(UTF_8));
|
||||
action.run();
|
||||
verifyNoInteractions(driveConnection);
|
||||
|
|
|
|||
|
|
@ -15,16 +15,13 @@
|
|||
package google.registry.reporting.icann;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.bigquery.model.TableFieldSchema;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableTable;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
|
|
@ -32,6 +29,7 @@ import google.registry.bigquery.BigqueryConnection;
|
|||
import google.registry.bigquery.BigqueryConnection.DestinationTable;
|
||||
import google.registry.bigquery.BigqueryUtils.TableType;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.reporting.icann.IcannReportingModule.ReportType;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeResponse;
|
||||
|
|
@ -46,9 +44,9 @@ class IcannReportingStagerTest {
|
|||
|
||||
private BigqueryConnection bigquery = mock(BigqueryConnection.class);
|
||||
FakeResponse response = new FakeResponse();
|
||||
private GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private YearMonth yearMonth = new YearMonth(2017, 6);
|
||||
private String subdir = "icann/monthly/2017-06";
|
||||
private GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
@RegisterExtension
|
||||
final AppEngineExtension appEngine =
|
||||
|
|
@ -65,7 +63,7 @@ class IcannReportingStagerTest {
|
|||
action.transactionsQueryBuilder = transactionsBuilder;
|
||||
action.reportingBucket = "test-bucket";
|
||||
action.bigquery = bigquery;
|
||||
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
action.gcsUtils = gcsUtils;
|
||||
return action;
|
||||
}
|
||||
|
||||
|
|
@ -100,14 +98,12 @@ class IcannReportingStagerTest {
|
|||
String expectedReport1 = "fooField,barField\r\n12,34";
|
||||
String expectedReport2 = "fooField,barField\r\n56,78";
|
||||
byte[] generatedFile1 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-activity-201706.csv"));
|
||||
assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
|
||||
byte[] generatedFile2 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-activity-201706.csv"));
|
||||
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
|
||||
}
|
||||
|
||||
|
|
@ -144,14 +140,12 @@ class IcannReportingStagerTest {
|
|||
"registrar,iana,field\r\n\"reg1\",123,10\r\n\"reg2\",456,20\r\nTotals,,30";
|
||||
String expectedReport2 = "registrar,iana,field\r\n\"reg1\",123,30\r\nTotals,,30";
|
||||
byte[] generatedFile1 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "fooTld-transactions-201706.csv"));
|
||||
assertThat(new String(generatedFile1, UTF_8)).isEqualTo(expectedReport1);
|
||||
byte[] generatedFile2 =
|
||||
readGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
|
||||
gcsUtils.readBytesFrom(
|
||||
BlobId.of("test-bucket/icann/monthly/2017-06", "barTld-transactions-201706.csv"));
|
||||
assertThat(new String(generatedFile2, UTF_8)).isEqualTo(expectedReport2);
|
||||
}
|
||||
|
||||
|
|
@ -164,8 +158,7 @@ class IcannReportingStagerTest {
|
|||
|
||||
String expectedManifest = "fooTld-transactions-201706.csv\nbarTld-activity-201706.csv\n";
|
||||
byte[] generatedManifest =
|
||||
readGcsFile(
|
||||
gcsService, new GcsFilename("test-bucket/icann/monthly/2017-06", "MANIFEST.txt"));
|
||||
gcsUtils.readBytesFrom(BlobId.of("test-bucket/icann/monthly/2017-06", "MANIFEST.txt"));
|
||||
assertThat(new String(generatedManifest, UTF_8)).isEqualTo(expectedManifest);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
|||
import static google.registry.testing.DatabaseHelper.createTlds;
|
||||
import static google.registry.testing.DatabaseHelper.loadByKey;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.writeGcsFile;
|
||||
import static google.registry.testing.LogsSubject.assertAboutLogs;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
|
@ -29,11 +28,10 @@ import static org.mockito.Mockito.verify;
|
|||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.testing.TestLogHandler;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.common.Cursor.CursorType;
|
||||
import google.registry.model.registry.Registry;
|
||||
|
|
@ -69,7 +67,7 @@ class IcannReportingUploadActionTest {
|
|||
private final IcannHttpReporter mockReporter = mock(IcannHttpReporter.class);
|
||||
private final SendEmailService emailService = mock(SendEmailService.class);
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final GcsService gcsService = GcsServiceFactory.createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
private final TestLogHandler logHandler = new TestLogHandler();
|
||||
private final Logger loggerToIntercept =
|
||||
Logger.getLogger(IcannReportingUploadAction.class.getCanonicalName());
|
||||
|
|
@ -78,7 +76,7 @@ class IcannReportingUploadActionTest {
|
|||
private IcannReportingUploadAction createAction() throws Exception {
|
||||
IcannReportingUploadAction action = new IcannReportingUploadAction();
|
||||
action.icannReporter = mockReporter;
|
||||
action.gcsUtils = new GcsUtils(gcsService, 1024);
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
|
||||
action.reportingBucket = "basin";
|
||||
action.emailService = emailService;
|
||||
|
|
@ -93,22 +91,14 @@ class IcannReportingUploadActionTest {
|
|||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
createTlds("tld", "foo");
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
|
||||
PAYLOAD_FAIL);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "foo-activity-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "tld-transactions-200606.csv"), PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_FAIL);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "foo-transactions-200606.csv"), PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "foo-activity-200606.csv"), PAYLOAD_SUCCESS);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200606.csv")).thenReturn(true);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "foo-transactions-200606.csv")).thenReturn(true);
|
||||
when(mockReporter.send(PAYLOAD_FAIL, "tld-activity-200606.csv")).thenReturn(false);
|
||||
|
|
@ -161,14 +151,10 @@ class IcannReportingUploadActionTest {
|
|||
persistResource(
|
||||
Cursor.create(
|
||||
CursorType.ICANN_UPLOAD_TX, DateTime.parse("2006-01-01TZ"), Registry.get("tld")));
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2005-12", "tld-activity-200512.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2005-12", "tld-transactions-200512.csv"), PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2005-12", "tld-activity-200512.csv"), PAYLOAD_SUCCESS);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200512.csv")).thenReturn(true);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-transactions-200512.csv")).thenReturn(true);
|
||||
|
||||
|
|
@ -191,10 +177,8 @@ class IcannReportingUploadActionTest {
|
|||
|
||||
@TestOfyAndSql
|
||||
void testSuccess_advancesCursor() throws Exception {
|
||||
writeGcsFile(
|
||||
gcsService,
|
||||
new GcsFilename("basin/icann/monthly/2006-06", "tld-activity-200606.csv"),
|
||||
PAYLOAD_SUCCESS);
|
||||
gcsUtils.createFromBytes(
|
||||
BlobId.of("basin/icann/monthly/2006-06", "tld-activity-200606.csv"), PAYLOAD_SUCCESS);
|
||||
when(mockReporter.send(PAYLOAD_SUCCESS, "tld-activity-200606.csv")).thenReturn(true);
|
||||
IcannReportingUploadAction action = createAction();
|
||||
action.run();
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import static com.google.common.truth.Truth8.assertThat;
|
|||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
|
@ -133,8 +133,8 @@ public class Spec11RegistrarThreatMatchesParserTest {
|
|||
}
|
||||
|
||||
private void setupFile(String fileWithContent, String fileDate) {
|
||||
GcsFilename gcsFilename =
|
||||
new GcsFilename(
|
||||
BlobId gcsFilename =
|
||||
BlobId.of(
|
||||
"test-bucket",
|
||||
String.format("icann/spec11/2018-07/SPEC11_MONTHLY_REPORT_%s", fileDate));
|
||||
when(gcsUtils.existsAndNotEmpty(gcsFilename)).thenReturn(true);
|
||||
|
|
|
|||
|
|
@ -1,48 +0,0 @@
|
|||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.testing;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFileOptions;
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.common.io.ByteStreams;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.Channels;
|
||||
|
||||
/** Utility methods for testing Google Cloud Storage code. */
|
||||
public final class GcsTestingUtils {
|
||||
|
||||
/** Slurps a Cloud Storage file into memory. */
|
||||
public static byte[] readGcsFile(GcsService gcsService, GcsFilename file)
|
||||
throws IOException {
|
||||
try (InputStream input = Channels.newInputStream(gcsService.openReadChannel(file, 0))) {
|
||||
return ByteStreams.toByteArray(input);
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes a Cloud Storage file. */
|
||||
public static void writeGcsFile(GcsService gcsService, GcsFilename file, byte[] data)
|
||||
throws IOException {
|
||||
gcsService.createOrReplace(file, GcsFileOptions.getDefaultInstance(), ByteBuffer.wrap(data));
|
||||
}
|
||||
|
||||
public static void deleteGcsFile(GcsService gcsService, GcsFilename file) throws IOException {
|
||||
gcsService.delete(file);
|
||||
}
|
||||
|
||||
private GcsTestingUtils() {}
|
||||
}
|
||||
|
|
@ -14,7 +14,6 @@
|
|||
|
||||
package google.registry.tools.server;
|
||||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.DatabaseHelper.createTlds;
|
||||
import static google.registry.testing.DatabaseHelper.newDomainBase;
|
||||
|
|
@ -23,17 +22,17 @@ import static google.registry.testing.DatabaseHelper.persistActiveContact;
|
|||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveHost;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static google.registry.testing.GcsTestingUtils.readGcsFile;
|
||||
import static google.registry.testing.TestDataHelper.loadFile;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.Duration.standardDays;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.common.base.Splitter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.gcs.backport.LocalStorageHelper;
|
||||
import google.registry.model.domain.secdns.DelegationSignerData;
|
||||
import google.registry.model.eppcommon.StatusValue;
|
||||
import google.registry.model.host.HostResource;
|
||||
|
|
@ -50,7 +49,7 @@ import org.junit.jupiter.api.Test;
|
|||
/** Tests for {@link GenerateZoneFilesAction}. */
|
||||
class GenerateZoneFilesActionTest extends MapreduceTestCase<GenerateZoneFilesAction> {
|
||||
|
||||
private final GcsService gcsService = createGcsService();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
|
||||
@Test
|
||||
void testGenerate() throws Exception {
|
||||
|
|
@ -114,7 +113,7 @@ class GenerateZoneFilesActionTest extends MapreduceTestCase<GenerateZoneFilesAct
|
|||
GenerateZoneFilesAction action = new GenerateZoneFilesAction();
|
||||
action.mrRunner = makeDefaultRunner();
|
||||
action.bucket = "zonefiles-bucket";
|
||||
action.gcsBufferSize = 123;
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.datastoreRetention = standardDays(29);
|
||||
action.dnsDefaultATtl = Duration.standardSeconds(11);
|
||||
action.dnsDefaultNsTtl = Duration.standardSeconds(222);
|
||||
|
|
@ -134,9 +133,8 @@ class GenerateZoneFilesActionTest extends MapreduceTestCase<GenerateZoneFilesAct
|
|||
|
||||
executeTasksUntilEmpty("mapreduce");
|
||||
|
||||
GcsFilename gcsFilename =
|
||||
new GcsFilename("zonefiles-bucket", String.format("tld-%s.zone", now));
|
||||
String generatedFile = new String(readGcsFile(gcsService, gcsFilename), UTF_8);
|
||||
BlobId gcsFilename = BlobId.of("zonefiles-bucket", String.format("tld-%s.zone", now));
|
||||
String generatedFile = new String(gcsUtils.readBytesFrom(gcsFilename), UTF_8);
|
||||
// The generated file contains spaces and tabs, but the golden file contains only spaces, as
|
||||
// files with literal tabs irritate our build tools.
|
||||
Splitter splitter = Splitter.on('\n').omitEmptyStrings();
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ com.google.code.findbugs:jsr305:3.0.2
|
|||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
|||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.j2objc:j2objc-annotations:1.3
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ com.google.errorprone:error_prone_annotations:2.5.1
|
|||
com.google.flogger:flogger-system-backend:0.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.38.0
|
||||
com.google.http-client:google-http-client-gson:1.38.1
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ ext {
|
|||
'com.google.apis:google-api-services-groupssettings:v1-rev82-1.25.0',
|
||||
'com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0',
|
||||
'com.google.apis:google-api-services-sheets:v4-rev612-1.25.0',
|
||||
'com.google.apis:google-api-services-storage:v1-rev171-1.25.0',
|
||||
'com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0',
|
||||
'com.google.appengine.tools:appengine-gcs-client:0.8.1',
|
||||
'com.google.appengine.tools:appengine-mapreduce:0.9',
|
||||
'com.google.appengine.tools:appengine-pipeline:0.2.13',
|
||||
|
|
@ -59,8 +59,10 @@ ext {
|
|||
'com.google.cloud.sql:jdbc-socket-factory-core:1.2.1',
|
||||
'com.google.cloud.sql:postgres-socket-factory:1.2.1',
|
||||
'com.google.cloud:google-cloud-core:1.94.3',
|
||||
'com.google.cloud:google-cloud-core-http:1.94.3',
|
||||
'com.google.cloud:google-cloud-secretmanager:1.4.0',
|
||||
'com.google.cloud:google-cloud-storage:1.113.12',
|
||||
'com.google.cloud:google-cloud-nio:0.123.2',
|
||||
'com.google.code.findbugs:jsr305:3.0.2',
|
||||
'com.google.code.gson:gson:2.8.6',
|
||||
'com.google.dagger:dagger-compiler:2.33',
|
||||
|
|
@ -68,8 +70,8 @@ ext {
|
|||
'com.google.errorprone:error_prone_annotations:2.5.1',
|
||||
'com.google.flogger:flogger-system-backend:0.5.1',
|
||||
'com.google.flogger:flogger:0.5.1',
|
||||
'com.google.guava:guava-testlib:30.1-jre',
|
||||
'com.google.guava:guava:30.1-jre',
|
||||
'com.google.guava:guava-testlib:30.1.1-jre',
|
||||
'com.google.guava:guava:30.1.1-jre',
|
||||
'com.google.gwt:gwt-user:2.9.0',
|
||||
'com.google.http-client:google-http-client-appengine:1.39.0',
|
||||
'com.google.http-client:google-http-client-jackson2:1.39.0',
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -112,7 +113,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -132,8 +133,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -222,7 +223,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.conscrypt:conscrypt-openjdk-uber:2.5.1
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -232,7 +233,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.googlecode.charts4j:charts4j:1.3
|
||||
|
|
@ -231,7 +232,7 @@ org.bouncycastle:bcpg-jdk15on:1.61
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.5
|
||||
org.checkerframework:checker-qual:3.7.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.codehaus.jackson:jackson-core-asl:1.9.13
|
||||
org.codehaus.jackson:jackson-mapper-asl:1.9.13
|
||||
org.codehaus.mojo:animal-sniffer-annotations:1.20
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -54,10 +54,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -75,7 +75,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -96,12 +96,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -112,7 +113,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -132,8 +133,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.14.0
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -59,10 +59,10 @@ com.google.api.grpc:proto-google-cloud-spanner-admin-database-v1:2.0.2
|
|||
com.google.api.grpc:proto-google-cloud-spanner-admin-instance-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-cloud-spanner-v1:2.0.2
|
||||
com.google.api.grpc:proto-google-common-protos:2.1.0
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.9
|
||||
com.google.api.grpc:proto-google-iam-v1:1.0.10
|
||||
com.google.api:api-common:1.10.1
|
||||
com.google.api:gax-grpc:1.62.0
|
||||
com.google.api:gax-httpjson:0.76.1
|
||||
com.google.api:gax-httpjson:0.79.0
|
||||
com.google.api:gax:1.62.0
|
||||
com.google.apis:google-api-services-admin-directory:directory_v1-rev118-1.25.0
|
||||
com.google.apis:google-api-services-appengine:v1-rev130-1.25.0
|
||||
|
|
@ -80,7 +80,7 @@ com.google.apis:google-api-services-monitoring:v3-rev540-1.25.0
|
|||
com.google.apis:google-api-services-pubsub:v1-rev20200713-1.30.10
|
||||
com.google.apis:google-api-services-sheets:v4-rev612-1.25.0
|
||||
com.google.apis:google-api-services-sqladmin:v1beta4-rev20210119-1.31.0
|
||||
com.google.apis:google-api-services-storage:v1-rev20200927-1.30.10
|
||||
com.google.apis:google-api-services-storage:v1-rev20210127-1.31.0
|
||||
com.google.appengine.tools:appengine-gcs-client:0.8.1
|
||||
com.google.appengine.tools:appengine-mapreduce:0.9
|
||||
com.google.appengine.tools:appengine-pipeline:0.2.13
|
||||
|
|
@ -102,12 +102,13 @@ com.google.cloud:google-cloud-bigquery:1.122.2
|
|||
com.google.cloud:google-cloud-bigquerystorage:1.5.5
|
||||
com.google.cloud:google-cloud-bigtable:1.14.0
|
||||
com.google.cloud:google-cloud-core-grpc:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.93.9
|
||||
com.google.cloud:google-cloud-core:1.93.9
|
||||
com.google.cloud:google-cloud-core-http:1.94.1
|
||||
com.google.cloud:google-cloud-core:1.94.3
|
||||
com.google.cloud:google-cloud-pubsub:1.110.0
|
||||
com.google.cloud:google-cloud-pubsublite:0.7.0
|
||||
com.google.cloud:google-cloud-secretmanager:1.4.0
|
||||
com.google.cloud:google-cloud-spanner:2.0.2
|
||||
com.google.cloud:google-cloud-storage:1.113.12
|
||||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.common.html.types:types:1.0.6
|
||||
|
|
@ -118,7 +119,7 @@ com.google.flogger:flogger-system-backend:0.5.1
|
|||
com.google.flogger:flogger:0.5.1
|
||||
com.google.flogger:google-extensions:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.gwt:gwt-user:2.9.0
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
|
|
@ -138,8 +139,8 @@ com.google.oauth-client:google-oauth-client-java6:1.31.4
|
|||
com.google.oauth-client:google-oauth-client-jetty:1.31.4
|
||||
com.google.oauth-client:google-oauth-client-servlet:1.31.4
|
||||
com.google.oauth-client:google-oauth-client:1.31.4
|
||||
com.google.protobuf:protobuf-java-util:3.15.2
|
||||
com.google.protobuf:protobuf-java:3.15.2
|
||||
com.google.protobuf:protobuf-java-util:3.15.3
|
||||
com.google.protobuf:protobuf-java:3.15.3
|
||||
com.google.re2j:re2j:1.6
|
||||
com.google.template:soy:2021-02-01
|
||||
com.google.truth:truth:1.1.2
|
||||
|
|
|
|||
|
|
@ -11,10 +11,10 @@ com.google.auto.value:auto-value:1.7.4
|
|||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
|
|
@ -45,5 +45,5 @@ org.apache.httpcomponents:httpcore:4.4.14
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.yaml:snakeyaml:1.17
|
||||
|
|
|
|||
|
|
@ -11,10 +11,10 @@ com.google.auto.value:auto-value:1.7.4
|
|||
com.google.code.findbugs:jsr305:3.0.2
|
||||
com.google.code.gson:gson:2.8.6
|
||||
com.google.dagger:dagger:2.33
|
||||
com.google.errorprone:error_prone_annotations:2.3.4
|
||||
com.google.errorprone:error_prone_annotations:2.5.1
|
||||
com.google.flogger:flogger:0.5.1
|
||||
com.google.guava:failureaccess:1.0.1
|
||||
com.google.guava:guava:30.1-jre
|
||||
com.google.guava:guava:30.1.1-jre
|
||||
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
|
||||
com.google.http-client:google-http-client-apache-v2:1.39.0
|
||||
com.google.http-client:google-http-client-gson:1.39.0
|
||||
|
|
@ -45,5 +45,5 @@ org.apache.httpcomponents:httpcore:4.4.14
|
|||
org.bouncycastle:bcpkix-jdk15on:1.61
|
||||
org.bouncycastle:bcprov-jdk15on:1.61
|
||||
org.checkerframework:checker-compat-qual:2.5.3
|
||||
org.checkerframework:checker-qual:3.5.0
|
||||
org.checkerframework:checker-qual:3.8.0
|
||||
org.yaml:snakeyaml:1.17
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue