Add helper methods to DatastoreAdmin Operation object

These are needed by Datastore export management actions.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=224242281

java/google/registry/export/datastore/BUILD

@@ -9,10 +9,13 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//java/google/registry/config",
+        "//java/google/registry/util",
         "@com_google_api_client",
+        "@com_google_code_findbugs_jsr305",
         "@com_google_dagger",
         "@com_google_guava",
         "@com_google_http_client",
         "@com_google_http_client_jackson2",
+        "@joda_time",
     ],
 )

java/google/registry/export/datastore/Operation.java

@@ -20,10 +20,20 @@ import com.google.api.client.json.GenericJson;
 import com.google.api.client.util.Key;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.export.datastore.DatastoreAdmin.Get;
+import google.registry.util.Clock;
 import java.util.List;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
 
-/** Model object that describes the details of an export or import operation in Cloud Datastore. */
+/**
+ * Model object that describes the details of an export or import operation in Cloud Datastore.
+ *
+ * <p>{@link Operation} instances are parsed from the JSON payload in Datastore response messages.
+ */
 public class Operation extends GenericJson {
 
   private static final String STATE_SUCCESS = "SUCCESSFUL";
@@ -46,24 +56,78 @@ public class Operation extends GenericJson {
     return done;
   }
 
-  public String getState() {
+  private String getState() {
-    checkState(metadata != null, "Response metadata missing.");
+    return getMetadata().getCommonMetadata().getState();
-    return metadata.getCommonMetadata().getState();
   }
 
   public boolean isSuccessful() {
-    checkState(metadata != null, "Response metadata missing.");
     return getState().equals(STATE_SUCCESS);
   }
 
   public boolean isProcessing() {
-    checkState(metadata != null, "Response metadata missing.");
     return getState().equals(STATE_PROCESSING);
   }
 
+  public Duration getRunningTime(Clock clock) {
+    return new Duration(
+        getStartTime(), getMetadata().getCommonMetadata().getEndTime().orElse(clock.nowUtc()));
+  }
+
+  public DateTime getStartTime() {
+    return getMetadata().getCommonMetadata().getStartTime();
+  }
+
+  public ImmutableSet<String> getKinds() {
+    return ImmutableSet.copyOf(getMetadata().getEntityFilter().getKinds());
+  }
+
+  /**
+   * Returns the URL to the GCS folder that holds the exported data. This folder is created by
+   * Datastore and is under the {@code outputUrlPrefix} set to {@linkplain
+   * DatastoreAdmin#export(String, List) the export request}.
+   */
+  public String getExportFolderUrl() {
+    return getMetadata().getOutputUrlPrefix();
+  }
+
+  /**
+   * Returns the last segment of the {@linkplain #getExportFolderUrl() export folder URL} which can
+   * be used as unique identifier of this export operation. This is a better ID than the {@linkplain
+   * #getName() operation name}, which is opaque.
+   */
+  public String getExportId() {
+    String exportFolderUrl = getExportFolderUrl();
+    return exportFolderUrl.substring(exportFolderUrl.lastIndexOf('/') + 1);
+  }
+
+  public String getProgress() {
+    StringBuilder result = new StringBuilder();
+    Progress progress = getMetadata().getProgressBytes();
+    if (progress != null) {
+      result.append(
+          String.format(" [%s/%s bytes]", progress.workCompleted, progress.workEstimated));
+    }
+    progress = getMetadata().getProgressEntities();
+    if (progress != null) {
+      result.append(
+          String.format(" [%s/%s entities]", progress.workCompleted, progress.workEstimated));
+    }
+    if (result.length() == 0) {
+      return "Progress: N/A";
+    }
+    return "Progress:" + result;
+  }
+
+  private Metadata getMetadata() {
+    checkState(metadata != null, "Response metadata missing.");
+    return metadata;
+  }
+
   /** Models the common metadata properties of all operations. */
   public static class CommonMetadata extends GenericJson {
 
+    @Key private String startTime;
+    @Key @Nullable private String endTime;
     @Key private String operationType;
     @Key private String state;
 
@@ -78,6 +142,15 @@ public class Operation extends GenericJson {
       checkState(!Strings.isNullOrEmpty(state), "state may not be null or empty");
       return state;
     }
+
+    DateTime getStartTime() {
+      checkState(startTime != null, "StartTime missing.");
+      return DateTime.parse(startTime);
+    }
+
+    Optional<DateTime> getEndTime() {
+      return Optional.ofNullable(endTime).map(DateTime::parse);
+    }
   }
 
   /** Models the metadata of a Cloud Datatore export or import operation. */
@@ -110,6 +183,7 @@ public class Operation extends GenericJson {
     }
 
     public String getOutputUrlPrefix() {
+      checkState(!Strings.isNullOrEmpty(outputUrlPrefix), "outputUrlPrefix");
       return outputUrlPrefix;
     }
   }

javatests/google/registry/export/datastore/BUILD

@@ -13,6 +13,7 @@ java_library(
     resources = glob(["**/testdata/*.json"]),
     deps = [
         "//java/google/registry/export/datastore",
+        "//java/google/registry/util",
         "//javatests/google/registry/testing",
         "@com_google_api_client",
         "@com_google_guava",
@@ -20,6 +21,7 @@ java_library(
         "@com_google_http_client_jackson2",
         "@com_google_truth",
         "@com_google_truth_extensions_truth_java8_extension",
+        "@joda_time",
         "@junit",
         "@org_mockito_all",
     ],

javatests/google/registry/export/datastore/DatastoreAdminTest.java

@@ -134,6 +134,23 @@ public class DatastoreAdminTest {
     assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
   }
 
+  @Test
+  public void testListOperations_filterByState() throws IOException {
+    // TODO(weiminyu): consider adding a method to DatastoreAdmin to support query by state.
+    DatastoreAdmin.ListOperations listOperations =
+        datastoreAdmin.list("metadata.common.state=PROCESSING");
+    HttpRequest httpRequest = listOperations.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo(
+            "https://datastore.googleapis.com/v1/projects/MyCloudProject/operations"
+                + "?filter=metadata.common.state%3DPROCESSING");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("GET");
+    assertThat(httpRequest.getContent()).isNull();
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
   private static HttpRequest simulateSendRequest(HttpRequest httpRequest) {
     try {
       httpRequest.setUrl(new GenericUrl("https://localhost:65537")).execute();

javatests/google/registry/export/datastore/OperationTest.java

@@ -15,14 +15,19 @@
 package google.registry.export.datastore;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
 
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import google.registry.export.datastore.Operation.CommonMetadata;
 import google.registry.export.datastore.Operation.Metadata;
 import google.registry.export.datastore.Operation.Progress;
+import google.registry.testing.FakeClock;
 import google.registry.testing.TestDataHelper;
+import google.registry.util.Clock;
 import java.io.IOException;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -37,6 +42,9 @@ public class OperationTest {
     CommonMetadata commonMetadata = loadJson("common_metadata.json", CommonMetadata.class);
     assertThat(commonMetadata.getState()).isEqualTo("SUCCESSFUL");
     assertThat(commonMetadata.getOperationType()).isEqualTo("EXPORT_ENTITIES");
+    assertThat(commonMetadata.getStartTime())
+        .isEqualTo(DateTime.parse("2018-10-29T16:01:04.645299Z"));
+    assertThat(commonMetadata.getEndTime()).isEmpty();
   }
 
   @Test
@@ -51,6 +59,12 @@ public class OperationTest {
     Metadata metadata = loadJson("metadata.json", Metadata.class);
     assertThat(metadata.getCommonMetadata().getOperationType()).isEqualTo("EXPORT_ENTITIES");
     assertThat(metadata.getCommonMetadata().getState()).isEqualTo("SUCCESSFUL");
+    assertThat(metadata.getCommonMetadata().getStartTime())
+        .isEqualTo(DateTime.parse("2018-10-29T16:01:04.645299Z"));
+    assertThat(metadata.getCommonMetadata().getEndTime())
+        .hasValue(DateTime.parse("2018-10-29T16:02:19.009859Z"));
+    assertThat(metadata.getOutputUrlPrefix())
+        .isEqualTo("gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364");
   }
 
   @Test
@@ -61,6 +75,15 @@ public class OperationTest {
     assertThat(operation.isProcessing()).isTrue();
     assertThat(operation.isSuccessful()).isFalse();
     assertThat(operation.isDone()).isFalse();
+    assertThat(operation.getStartTime()).isEqualTo(DateTime.parse("2018-10-29T16:01:04.645299Z"));
+    assertThat(operation.getExportFolderUrl())
+        .isEqualTo("gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364");
+    assertThat(operation.getExportId()).isEqualTo("2018-10-29T16:01:04_99364");
+    assertThat(operation.getKinds()).containsExactly("Registry", "Registrar", "DomainBase");
+    assertThat(operation.toPrettyString())
+        .isEqualTo(
+            TestDataHelper.loadFile(OperationTest.class, "prettyprinted_operation.json").trim());
+    assertThat(operation.getProgress()).isEqualTo("Progress: N/A");
   }
 
   @Test
@@ -68,6 +91,16 @@ public class OperationTest {
     Operation.OperationList operationList =
         loadJson("operation_list.json", Operation.OperationList.class);
     assertThat(operationList.toList()).hasSize(2);
+    Clock clock = new FakeClock(DateTime.parse("2018-10-29T16:01:04.645299Z"));
+    ((FakeClock) clock).advanceOneMilli();
+    assertThat(operationList.toList().get(0).getRunningTime(clock)).isEqualTo(Duration.millis(1));
+    assertThat(operationList.toList().get(0).getProgress())
+        .isEqualTo("Progress: [51797/54513 entities]");
+    assertThat(operationList.toList().get(1).getRunningTime(clock))
+        .isEqualTo(Duration.standardMinutes(1));
+    // Work completed may exceed work estimated
+    assertThat(operationList.toList().get(1).getProgress())
+        .isEqualTo("Progress: [96908367/73773755 bytes] [51797/54513 entities]");
   }
 
   private static <T> T loadJson(String fileName, Class<T> type) throws IOException {

javatests/google/registry/export/datastore/testdata/common_metadata.json

@@ -1,6 +1,5 @@
 {
     "startTime": "2018-10-29T16:01:04.645299Z",
-    "endTime": "2018-10-29T16:02:19.009859Z",
     "operationType": "EXPORT_ENTITIES",
     "state": "SUCCESSFUL"
 }

javatests/google/registry/export/datastore/testdata/operation_list.json

@@ -9,6 +9,10 @@
           "operationType": "EXPORT_ENTITIES",
           "state": "PROCESSING"
         },
+        "progressEntities": {
+          "workCompleted": "51797",
+          "workEstimated": "54513"
+        },
         "entityFilter": {
           "kinds": [
             "Registry",
@@ -25,9 +29,18 @@
         "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
         "common": {
           "startTime": "2018-10-29T16:01:04.645299Z",
+          "endTime": "2018-10-29T16:02:04.645299Z",
           "operationType": "EXPORT_ENTITIES",
           "state": "PROCESSING"
         },
+        "progressEntities": {
+          "workCompleted": "51797",
+          "workEstimated": "54513"
+        },
+        "progressBytes": {
+          "workCompleted": "96908367",
+          "workEstimated": "73773755"
+        },
         "entityFilter": {
           "kinds": [
             "Registry",

javatests/google/registry/export/datastore/testdata/prettyprinted_operation.json

@@ -0,0 +1,16 @@
+{
+  "done" : false,
+  "metadata" : {
+    "common" : {
+      "operationType" : "EXPORT_ENTITIES",
+      "startTime" : "2018-10-29T16:01:04.645299Z",
+      "state" : "PROCESSING"
+    },
+    "entityFilter" : {
+      "kinds" : [ "Registry", "Registrar", "DomainBase" ]
+    },
+    "outputUrlPrefix" : "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364",
+    "@type" : "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata"
+  },
+  "name" : "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ"
+}