diff --git a/java/google/registry/export/datastore/BUILD b/java/google/registry/export/datastore/BUILD index 16781c943..adc95e5a9 100644 --- a/java/google/registry/export/datastore/BUILD +++ b/java/google/registry/export/datastore/BUILD @@ -9,10 +9,13 @@ java_library( srcs = glob(["*.java"]), deps = [ "//java/google/registry/config", + "//java/google/registry/util", "@com_google_api_client", + "@com_google_code_findbugs_jsr305", "@com_google_dagger", "@com_google_guava", "@com_google_http_client", "@com_google_http_client_jackson2", + "@joda_time", ], ) diff --git a/java/google/registry/export/datastore/Operation.java b/java/google/registry/export/datastore/Operation.java index a1948b162..b9a4cf8b4 100644 --- a/java/google/registry/export/datastore/Operation.java +++ b/java/google/registry/export/datastore/Operation.java @@ -20,10 +20,20 @@ import com.google.api.client.json.GenericJson; import com.google.api.client.util.Key; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import google.registry.export.datastore.DatastoreAdmin.Get; +import google.registry.util.Clock; import java.util.List; +import java.util.Optional; +import javax.annotation.Nullable; +import org.joda.time.DateTime; +import org.joda.time.Duration; -/** Model object that describes the details of an export or import operation in Cloud Datastore. */ +/** + * Model object that describes the details of an export or import operation in Cloud Datastore. + * + *

{@link Operation} instances are parsed from the JSON payload in Datastore response messages. + */ public class Operation extends GenericJson { private static final String STATE_SUCCESS = "SUCCESSFUL"; @@ -46,24 +56,78 @@ public class Operation extends GenericJson { return done; } - public String getState() { - checkState(metadata != null, "Response metadata missing."); - return metadata.getCommonMetadata().getState(); + private String getState() { + return getMetadata().getCommonMetadata().getState(); } public boolean isSuccessful() { - checkState(metadata != null, "Response metadata missing."); return getState().equals(STATE_SUCCESS); } public boolean isProcessing() { - checkState(metadata != null, "Response metadata missing."); return getState().equals(STATE_PROCESSING); } + public Duration getRunningTime(Clock clock) { + return new Duration( + getStartTime(), getMetadata().getCommonMetadata().getEndTime().orElse(clock.nowUtc())); + } + + public DateTime getStartTime() { + return getMetadata().getCommonMetadata().getStartTime(); + } + + public ImmutableSet getKinds() { + return ImmutableSet.copyOf(getMetadata().getEntityFilter().getKinds()); + } + + /** + * Returns the URL to the GCS folder that holds the exported data. This folder is created by + * Datastore and is under the {@code outputUrlPrefix} set to {@linkplain + * DatastoreAdmin#export(String, List) the export request}. + */ + public String getExportFolderUrl() { + return getMetadata().getOutputUrlPrefix(); + } + + /** + * Returns the last segment of the {@linkplain #getExportFolderUrl() export folder URL} which can + * be used as unique identifier of this export operation. This is a better ID than the {@linkplain + * #getName() operation name}, which is opaque. + */ + public String getExportId() { + String exportFolderUrl = getExportFolderUrl(); + return exportFolderUrl.substring(exportFolderUrl.lastIndexOf('/') + 1); + } + + public String getProgress() { + StringBuilder result = new StringBuilder(); + Progress progress = getMetadata().getProgressBytes(); + if (progress != null) { + result.append( + String.format(" [%s/%s bytes]", progress.workCompleted, progress.workEstimated)); + } + progress = getMetadata().getProgressEntities(); + if (progress != null) { + result.append( + String.format(" [%s/%s entities]", progress.workCompleted, progress.workEstimated)); + } + if (result.length() == 0) { + return "Progress: N/A"; + } + return "Progress:" + result; + } + + private Metadata getMetadata() { + checkState(metadata != null, "Response metadata missing."); + return metadata; + } + /** Models the common metadata properties of all operations. */ public static class CommonMetadata extends GenericJson { + @Key private String startTime; + @Key @Nullable private String endTime; @Key private String operationType; @Key private String state; @@ -78,6 +142,15 @@ public class Operation extends GenericJson { checkState(!Strings.isNullOrEmpty(state), "state may not be null or empty"); return state; } + + DateTime getStartTime() { + checkState(startTime != null, "StartTime missing."); + return DateTime.parse(startTime); + } + + Optional getEndTime() { + return Optional.ofNullable(endTime).map(DateTime::parse); + } } /** Models the metadata of a Cloud Datatore export or import operation. */ @@ -110,6 +183,7 @@ public class Operation extends GenericJson { } public String getOutputUrlPrefix() { + checkState(!Strings.isNullOrEmpty(outputUrlPrefix), "outputUrlPrefix"); return outputUrlPrefix; } } diff --git a/javatests/google/registry/export/datastore/BUILD b/javatests/google/registry/export/datastore/BUILD index 1161ff343..96fb88629 100644 --- a/javatests/google/registry/export/datastore/BUILD +++ b/javatests/google/registry/export/datastore/BUILD @@ -13,6 +13,7 @@ java_library( resources = glob(["**/testdata/*.json"]), deps = [ "//java/google/registry/export/datastore", + "//java/google/registry/util", "//javatests/google/registry/testing", "@com_google_api_client", "@com_google_guava", @@ -20,6 +21,7 @@ java_library( "@com_google_http_client_jackson2", "@com_google_truth", "@com_google_truth_extensions_truth_java8_extension", + "@joda_time", "@junit", "@org_mockito_all", ], diff --git a/javatests/google/registry/export/datastore/DatastoreAdminTest.java b/javatests/google/registry/export/datastore/DatastoreAdminTest.java index 12aae02c7..0e30aa2b3 100644 --- a/javatests/google/registry/export/datastore/DatastoreAdminTest.java +++ b/javatests/google/registry/export/datastore/DatastoreAdminTest.java @@ -134,6 +134,23 @@ public class DatastoreAdminTest { assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN); } + @Test + public void testListOperations_filterByState() throws IOException { + // TODO(weiminyu): consider adding a method to DatastoreAdmin to support query by state. + DatastoreAdmin.ListOperations listOperations = + datastoreAdmin.list("metadata.common.state=PROCESSING"); + HttpRequest httpRequest = listOperations.buildHttpRequest(); + assertThat(httpRequest.getUrl().toString()) + .isEqualTo( + "https://datastore.googleapis.com/v1/projects/MyCloudProject/operations" + + "?filter=metadata.common.state%3DPROCESSING"); + assertThat(httpRequest.getRequestMethod()).isEqualTo("GET"); + assertThat(httpRequest.getContent()).isNull(); + + simulateSendRequest(httpRequest); + assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN); + } + private static HttpRequest simulateSendRequest(HttpRequest httpRequest) { try { httpRequest.setUrl(new GenericUrl("https://localhost:65537")).execute(); diff --git a/javatests/google/registry/export/datastore/OperationTest.java b/javatests/google/registry/export/datastore/OperationTest.java index 0932aae3f..252b5dac3 100644 --- a/javatests/google/registry/export/datastore/OperationTest.java +++ b/javatests/google/registry/export/datastore/OperationTest.java @@ -15,14 +15,19 @@ package google.registry.export.datastore; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; import google.registry.export.datastore.Operation.CommonMetadata; import google.registry.export.datastore.Operation.Metadata; import google.registry.export.datastore.Operation.Progress; +import google.registry.testing.FakeClock; import google.registry.testing.TestDataHelper; +import google.registry.util.Clock; import java.io.IOException; +import org.joda.time.DateTime; +import org.joda.time.Duration; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -37,6 +42,9 @@ public class OperationTest { CommonMetadata commonMetadata = loadJson("common_metadata.json", CommonMetadata.class); assertThat(commonMetadata.getState()).isEqualTo("SUCCESSFUL"); assertThat(commonMetadata.getOperationType()).isEqualTo("EXPORT_ENTITIES"); + assertThat(commonMetadata.getStartTime()) + .isEqualTo(DateTime.parse("2018-10-29T16:01:04.645299Z")); + assertThat(commonMetadata.getEndTime()).isEmpty(); } @Test @@ -51,6 +59,12 @@ public class OperationTest { Metadata metadata = loadJson("metadata.json", Metadata.class); assertThat(metadata.getCommonMetadata().getOperationType()).isEqualTo("EXPORT_ENTITIES"); assertThat(metadata.getCommonMetadata().getState()).isEqualTo("SUCCESSFUL"); + assertThat(metadata.getCommonMetadata().getStartTime()) + .isEqualTo(DateTime.parse("2018-10-29T16:01:04.645299Z")); + assertThat(metadata.getCommonMetadata().getEndTime()) + .hasValue(DateTime.parse("2018-10-29T16:02:19.009859Z")); + assertThat(metadata.getOutputUrlPrefix()) + .isEqualTo("gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"); } @Test @@ -61,6 +75,15 @@ public class OperationTest { assertThat(operation.isProcessing()).isTrue(); assertThat(operation.isSuccessful()).isFalse(); assertThat(operation.isDone()).isFalse(); + assertThat(operation.getStartTime()).isEqualTo(DateTime.parse("2018-10-29T16:01:04.645299Z")); + assertThat(operation.getExportFolderUrl()) + .isEqualTo("gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"); + assertThat(operation.getExportId()).isEqualTo("2018-10-29T16:01:04_99364"); + assertThat(operation.getKinds()).containsExactly("Registry", "Registrar", "DomainBase"); + assertThat(operation.toPrettyString()) + .isEqualTo( + TestDataHelper.loadFile(OperationTest.class, "prettyprinted_operation.json").trim()); + assertThat(operation.getProgress()).isEqualTo("Progress: N/A"); } @Test @@ -68,6 +91,16 @@ public class OperationTest { Operation.OperationList operationList = loadJson("operation_list.json", Operation.OperationList.class); assertThat(operationList.toList()).hasSize(2); + Clock clock = new FakeClock(DateTime.parse("2018-10-29T16:01:04.645299Z")); + ((FakeClock) clock).advanceOneMilli(); + assertThat(operationList.toList().get(0).getRunningTime(clock)).isEqualTo(Duration.millis(1)); + assertThat(operationList.toList().get(0).getProgress()) + .isEqualTo("Progress: [51797/54513 entities]"); + assertThat(operationList.toList().get(1).getRunningTime(clock)) + .isEqualTo(Duration.standardMinutes(1)); + // Work completed may exceed work estimated + assertThat(operationList.toList().get(1).getProgress()) + .isEqualTo("Progress: [96908367/73773755 bytes] [51797/54513 entities]"); } private static T loadJson(String fileName, Class type) throws IOException { diff --git a/javatests/google/registry/export/datastore/testdata/common_metadata.json b/javatests/google/registry/export/datastore/testdata/common_metadata.json index ab77ea33a..7354fdec5 100644 --- a/javatests/google/registry/export/datastore/testdata/common_metadata.json +++ b/javatests/google/registry/export/datastore/testdata/common_metadata.json @@ -1,6 +1,5 @@ { "startTime": "2018-10-29T16:01:04.645299Z", - "endTime": "2018-10-29T16:02:19.009859Z", "operationType": "EXPORT_ENTITIES", "state": "SUCCESSFUL" } diff --git a/javatests/google/registry/export/datastore/testdata/operation_list.json b/javatests/google/registry/export/datastore/testdata/operation_list.json index 116774f62..7d464c083 100644 --- a/javatests/google/registry/export/datastore/testdata/operation_list.json +++ b/javatests/google/registry/export/datastore/testdata/operation_list.json @@ -9,6 +9,10 @@ "operationType": "EXPORT_ENTITIES", "state": "PROCESSING" }, + "progressEntities": { + "workCompleted": "51797", + "workEstimated": "54513" + }, "entityFilter": { "kinds": [ "Registry", @@ -25,9 +29,18 @@ "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata", "common": { "startTime": "2018-10-29T16:01:04.645299Z", + "endTime": "2018-10-29T16:02:04.645299Z", "operationType": "EXPORT_ENTITIES", "state": "PROCESSING" }, + "progressEntities": { + "workCompleted": "51797", + "workEstimated": "54513" + }, + "progressBytes": { + "workCompleted": "96908367", + "workEstimated": "73773755" + }, "entityFilter": { "kinds": [ "Registry", diff --git a/javatests/google/registry/export/datastore/testdata/prettyprinted_operation.json b/javatests/google/registry/export/datastore/testdata/prettyprinted_operation.json new file mode 100644 index 000000000..73f2f9c87 --- /dev/null +++ b/javatests/google/registry/export/datastore/testdata/prettyprinted_operation.json @@ -0,0 +1,16 @@ +{ + "done" : false, + "metadata" : { + "common" : { + "operationType" : "EXPORT_ENTITIES", + "startTime" : "2018-10-29T16:01:04.645299Z", + "state" : "PROCESSING" + }, + "entityFilter" : { + "kinds" : [ "Registry", "Registrar", "DomainBase" ] + }, + "outputUrlPrefix" : "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364", + "@type" : "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata" + }, + "name" : "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ" +}