Add a package to open source

Add {java,tests}/google/registry/export/datastore to open source.
This is part of the migration to Datastore Managed Import/Export
for backup.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=221800709

java/google/registry/export/datastore/BUILD

@@ -0,0 +1,21 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+java_library(
+    name = "datastore",
+    srcs = glob(["*.java"]),
+    deps = [
+        "//java/google/registry/config",
+        "//third_party/java/google_api_java_client:services_json",
+        "//third_party/java/google_http_java_client:http",
+        "//third_party/java/google_http_java_client:json",
+        "//third_party/java/google_http_java_client:util",
+        "@com_google_api_client",
+        "@com_google_dagger",
+        "@com_google_guava",
+        "@com_google_http_client_jackson2",
+    ],
+)

java/google/registry/export/datastore/DatastoreAdmin.java

@@ -0,0 +1,223 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.api.client.googleapis.services.json.AbstractGoogleJsonClient;
+import com.google.api.client.googleapis.services.json.AbstractGoogleJsonClientRequest;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.GenericJson;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.util.Key;
+import com.google.common.base.Strings;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Java client to <a href="https://cloud.google.com/datastore/docs/reference/admin/rest/">Cloud
+ * Datastore Admin REST API</a>.
+ */
+public class DatastoreAdmin extends AbstractGoogleJsonClient {
+
+  private static final String ROOT_URL = "https://datastore.googleapis.com/v1/";
+  private static final String SERVICE_PATH = "";
+
+  // GCP project that this instance is associated with.
+  private final String projectId;
+
+  protected DatastoreAdmin(Builder builder) {
+    super(builder);
+    this.projectId = checkNotNull(builder.projectId, "GCP projectId missing.");
+  }
+
+  /**
+   * Returns an {@link Export} request that starts exporting all Cloud Datastore databases owned by
+   * the GCP project identified by {@link #projectId}.
+   *
+   * <p>Typical usage is:
+   *
+   * <pre>
+   *     {@code Export export = datastoreAdmin.export(parameters ...);}
+   *     {@code Operation operation = export.execute();}
+   *     {@code while (!operation.isSuccessful()) { ...}}
+   * </pre>
+   *
+   * <p>Please see the <a
+   * href="https://cloud.google.com/datastore/docs/reference/admin/rest/v1/projects/export">API
+   * specification of the export method for details</a>.
+   *
+   * <p>The following undocumented behaviors with regard to {@code outputUrlPrefix} have been
+   * observed:
+   *
+   * <ul>
+   *   <li>If outputUrlPrefix refers to a GCS bucket, exported data will be nested deeper in the
+   *       bucket with a timestamped path. This is useful when periodical backups are desired
+   *   <li>If outputUrlPrefix is a already a nested path in a GCS bucket, exported data will be put
+   *       under this path. This means that a nested path is not reusable, since the export process
+   *       by default would not overwrite existing files.
+   * </ul>
+   *
+   * @param outputUrlPrefix the full resource URL of the external storage location
+   * @param kinds the datastore 'kinds' to be exported
+   */
+  public Export export(String outputUrlPrefix, List<String> kinds) {
+    return new Export(new ExportRequest(outputUrlPrefix, kinds));
+  }
+
+  /**
+   * Returns a {@link Get} request that retrieves the details of an export or import {@link
+   * Operation}.
+   *
+   * @param operationName name of the {@code Operation} as returned by an export or import request
+   */
+  public Get get(String operationName) {
+    return new Get(operationName);
+  }
+
+  /**
+   * Returns a {@link ListOperations} request that retrieves all export or import {@link Operation
+   * operations} matching {@code filter}.
+   *
+   * <p>Sample usage: find all operations started after 2018-10-31 00:00:00 UTC and has stopped:
+   *
+   * <pre>
+   *     {@code String filter = "metadata.common.startTime>\"2018-10-31T0:0:0Z\" AND done=true";}
+   *     {@code List<Operation> operations = datastoreAdmin.list(filter);}
+   * </pre>
+   *
+   * <p>Please refer to {@link Operation} for how to reference operation properties.
+   */
+  public ListOperations list(String filter) {
+    checkArgument(!Strings.isNullOrEmpty(filter), "Filter must not be null or empty.");
+    return new ListOperations(Optional.of(filter));
+  }
+
+  /**
+   * Returns a {@link ListOperations} request that retrieves all export or import {@link Operation *
+   * operations}.
+   */
+  public ListOperations listAll() {
+    return new ListOperations(Optional.empty());
+  }
+
+  /** Builder for {@link DatastoreAdmin}. */
+  public static class Builder extends AbstractGoogleJsonClient.Builder {
+
+    private String projectId;
+
+    public Builder(
+        HttpTransport httpTransport,
+        JsonFactory jsonFactory,
+        HttpRequestInitializer httpRequestInitializer) {
+      super(httpTransport, jsonFactory, ROOT_URL, SERVICE_PATH, httpRequestInitializer, false);
+    }
+
+    @Override
+    public Builder setApplicationName(String applicationName) {
+      return (Builder) super.setApplicationName(applicationName);
+    }
+
+    /** Sets the GCP project ID of the Cloud Datastore databases being managed. */
+    public Builder setProjectId(String projectId) {
+      this.projectId = projectId;
+      return this;
+    }
+
+    @Override
+    public DatastoreAdmin build() {
+      return new DatastoreAdmin(this);
+    }
+  }
+
+  /** A request to export Cloud Datastore databases. */
+  public class Export extends DatastoreAdminRequest<Operation> {
+
+    Export(ExportRequest exportRequest) {
+      super(
+          DatastoreAdmin.this,
+          "POST",
+          "projects/{projectId}:export",
+          exportRequest,
+          Operation.class);
+      set("projectId", projectId);
+    }
+  }
+
+  /** A request to retrieve details of an export or import operation. */
+  public class Get extends DatastoreAdminRequest<Operation> {
+
+    Get(String operationName) {
+      super(DatastoreAdmin.this, "GET", operationName, null, Operation.class);
+    }
+  }
+
+  /** A request to retrieve all export or import operations matching a given filter. */
+  public class ListOperations extends DatastoreAdminRequest<Operation.OperationList> {
+
+    ListOperations(Optional<String> filter) {
+      super(
+          DatastoreAdmin.this,
+          "GET",
+          "projects/{projectId}/operations",
+          null,
+          Operation.OperationList.class);
+      set("projectId", projectId);
+      filter.ifPresent(f -> set("filter", f));
+    }
+  }
+
+  /** Base class of all DatastoreAdmin requests. */
+  abstract static class DatastoreAdminRequest<T> extends AbstractGoogleJsonClientRequest<T> {
+    /**
+     * @param client Google JSON client
+     * @param requestMethod HTTP Method
+     * @param uriTemplate URI template for the path relative to the base URL. If it starts with a
+     *     "/" the base path from the base URL will be stripped out. The URI template can also be a
+     *     full URL. URI template expansion is done using {@link
+     *     com.google.api.client.http.UriTemplate#expand(String, String, Object, boolean)}
+     * @param jsonContent POJO that can be serialized into JSON content or {@code null} for none
+     * @param responseClass response class to parse into
+     */
+    protected DatastoreAdminRequest(
+        DatastoreAdmin client,
+        String requestMethod,
+        String uriTemplate,
+        Object jsonContent,
+        Class<T> responseClass) {
+      super(client, requestMethod, uriTemplate, jsonContent, responseClass);
+    }
+  }
+
+  /**
+   * Model object that describes the JSON content in an export request.
+   *
+   * <p>Please note that some properties defined in the API are excluded, e.g., {@code databaseId}
+   * (not supported by Cloud Datastore) and labels (not used by Domain Registry).
+   */
+  @SuppressWarnings("unused")
+  static class ExportRequest extends GenericJson {
+    @Key private final String outputUrlPrefix;
+    @Key private final EntityFilter entityFilter;
+
+    ExportRequest(String outputUrlPrefix, List<String> kinds) {
+      checkNotNull(outputUrlPrefix, "outputUrlPrefix");
+      this.outputUrlPrefix = outputUrlPrefix;
+      this.entityFilter = new EntityFilter(kinds);
+    }
+  }
+}

java/google/registry/export/datastore/DatastoreAdminModule.java

@@ -0,0 +1,39 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import javax.inject.Singleton;
+
+/** Dagger module that configures provision of {@link DatastoreAdmin}. */
+@Module
+public abstract class DatastoreAdminModule {
+
+  @Singleton
+  @Provides
+  static DatastoreAdmin provideDatastoreAdmin(
+      @CredentialModule.DefaultCredential GoogleCredential credential,
+      @RegistryConfig.Config("projectId") String projectId) {
+    return new DatastoreAdmin.Builder(
+            credential.getTransport(), credential.getJsonFactory(), credential)
+        .setApplicationName(projectId)
+        .setProjectId(projectId)
+        .build();
+  }
+}

java/google/registry/export/datastore/EntityFilter.java

@@ -0,0 +1,48 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.api.client.json.GenericJson;
+import com.google.api.client.util.Key;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+
+/**
+ * Model object that describes the Cloud Datastore 'kinds' to be exported or imported. The JSON form
+ * of this type is found in export/import requests and responses.
+ *
+ * <p>Please note that properties not used by Domain Registry are not included, e.g., {@code
+ * namespaceIds}.
+ */
+public class EntityFilter extends GenericJson {
+
+  @Key private List<String> kinds = ImmutableList.of();
+
+  /** For JSON deserialization. */
+  public EntityFilter() {}
+
+  EntityFilter(List<String> kinds) {
+    checkNotNull(kinds, "kinds");
+    checkArgument(!kinds.isEmpty(), "kinds must not be empty");
+    this.kinds = ImmutableList.copyOf(kinds);
+  }
+
+  List<String> getKinds() {
+    return ImmutableList.copyOf(kinds);
+  }
+}

java/google/registry/export/datastore/Operation.java

@@ -0,0 +1,144 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.api.client.json.GenericJson;
+import com.google.api.client.util.Key;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+import google.registry.export.datastore.DatastoreAdmin.Get;
+import java.util.List;
+
+/** Model object that describes the details of an export or import operation in Cloud Datastore. */
+public class Operation extends GenericJson {
+
+  private static final String STATE_SUCCESS = "SUCCESSFUL";
+  private static final String STATE_PROCESSING = "PROCESSING";
+
+  @Key private String name;
+  @Key private Metadata metadata;
+  @Key private boolean done;
+
+  /** For JSON deserialization. */
+  public Operation() {}
+
+  /** Returns the name of this operation, which may be used in a {@link Get} request. */
+  public String getName() {
+    checkState(name != null, "Name must not be null.");
+    return name;
+  }
+
+  public boolean isDone() {
+    return done;
+  }
+
+  public String getState() {
+    checkState(metadata != null, "Response metadata missing.");
+    return metadata.getCommonMetadata().getState();
+  }
+
+  public boolean isSuccessful() {
+    checkState(metadata != null, "Response metadata missing.");
+    return getState().equals(STATE_SUCCESS);
+  }
+
+  public boolean isProcessing() {
+    checkState(metadata != null, "Response metadata missing.");
+    return getState().equals(STATE_PROCESSING);
+  }
+
+  /** Models the common metadata properties of all operations. */
+  public static class CommonMetadata extends GenericJson {
+
+    @Key private String operationType;
+    @Key private String state;
+
+    public CommonMetadata() {}
+
+    String getOperationType() {
+      checkState(!Strings.isNullOrEmpty(operationType), "operationType may not be null or empty");
+      return operationType;
+    }
+
+    String getState() {
+      checkState(!Strings.isNullOrEmpty(state), "state may not be null or empty");
+      return state;
+    }
+  }
+
+  /** Models the metadata of a Cloud Datatore export or import operation. */
+  public static class Metadata extends GenericJson {
+    @Key("common")
+    private CommonMetadata commonMetadata;
+
+    @Key private Progress progressEntities;
+    @Key private Progress progressBytes;
+    @Key private EntityFilter entityFilter;
+    @Key private String outputUrlPrefix;
+
+    public Metadata() {}
+
+    CommonMetadata getCommonMetadata() {
+      checkState(commonMetadata != null, "CommonMetadata field is null.");
+      return commonMetadata;
+    }
+
+    public Progress getProgressEntities() {
+      return progressEntities;
+    }
+
+    public Progress getProgressBytes() {
+      return progressBytes;
+    }
+
+    public EntityFilter getEntityFilter() {
+      return entityFilter;
+    }
+
+    public String getOutputUrlPrefix() {
+      return outputUrlPrefix;
+    }
+  }
+
+  /** Progress of an export or import operation. */
+  public static class Progress extends GenericJson {
+    @Key private long workCompleted;
+    @Key private long workEstimated;
+
+    public Progress() {}
+
+    long getWorkCompleted() {
+      return workCompleted;
+    }
+
+    public long getWorkEstimated() {
+      return workEstimated;
+    }
+  }
+
+  /** List of {@link Operation Operations}. */
+  public static class OperationList extends GenericJson {
+    @Key private List<Operation> operations;
+
+    /** For JSON deserialization. */
+    public OperationList() {}
+
+    ImmutableList<Operation> toList() {
+      return ImmutableList.copyOf(operations);
+    }
+  }
+}

javatests/google/registry/export/datastore/BUILD

@@ -0,0 +1,35 @@
+package(
+    default_testonly = 1,
+    default_visibility = ["//java/google/registry:registry_project"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules")
+
+java_library(
+    name = "datastore",
+    srcs = glob(["*.java"]),
+    resources = glob(["**/testdata/*.json"]),
+    deps = [
+        "//java/google/registry/export/datastore",
+        "//javatests/google/registry/testing",
+        "//third_party/java/google_http_java_client:http",
+        "//third_party/java/google_http_java_client:javanet",
+        "//third_party/java/google_http_java_client:util",
+        "@com_google_api_client",
+        "@com_google_guava",
+        "@com_google_http_client",
+        "@com_google_http_client_jackson2",
+        "@com_google_truth",
+        "@com_google_truth_extensions_truth_java8_extension",
+        "@junit",
+        "@org_mockito_all",
+    ],
+)
+
+GenTestRules(
+    name = "GeneratedTestRules",
+    test_files = glob(["*Test.java"]),
+    deps = [":datastore"],
+)

javatests/google/registry/export/datastore/DatastoreAdminTest.java

@@ -0,0 +1,161 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.common.collect.ImmutableList;
+import google.registry.testing.MockitoJUnitRule;
+import google.registry.testing.TestDataHelper;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link DatastoreAdmin}. */
+@RunWith(JUnit4.class)
+public class DatastoreAdminTest {
+
+  private static final String AUTH_HEADER_PREFIX = "Bearer ";
+  private static final String ACCESS_TOKEN = "MyAccessToken";
+  private static final ImmutableList<String> KINDS =
+      ImmutableList.of("Registry", "Registrar", "DomainBase");
+
+  @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
+
+  private HttpTransport httpTransport;
+  private GoogleCredential googleCredential;
+  private DatastoreAdmin datastoreAdmin;
+
+  @Before
+  public void setup() {
+    httpTransport = new NetHttpTransport();
+    googleCredential =
+        new GoogleCredential.Builder()
+            .setTransport(httpTransport)
+            .setJsonFactory(JacksonFactory.getDefaultInstance())
+            .setClock(() -> 0)
+            .build();
+    googleCredential.setAccessToken(ACCESS_TOKEN);
+    googleCredential.setExpiresInSeconds(1_000L);
+
+    datastoreAdmin =
+        new DatastoreAdmin.Builder(
+                googleCredential.getTransport(),
+                googleCredential.getJsonFactory(),
+                googleCredential)
+            .setApplicationName("MyApplication")
+            .setProjectId("MyCloudProject")
+            .build();
+  }
+
+  @Test
+  public void testExport() throws IOException {
+    DatastoreAdmin.Export export = datastoreAdmin.export("gs://mybucket/path", KINDS);
+    HttpRequest httpRequest = export.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo("https://datastore.googleapis.com/v1/projects/MyCloudProject:export");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("POST");
+
+    assertThat(getRequestContent(httpRequest))
+        .hasValue(
+            TestDataHelper.loadFile(getClass(), "export_request_content.json")
+                .replaceAll("[\\s\\n]+", ""));
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  @Test
+  public void testGetOperation() throws IOException {
+    DatastoreAdmin.Get get =
+        datastoreAdmin.get("projects/MyCloudProject/operations/ASAzNjMwOTEyNjUJ");
+    HttpRequest httpRequest = get.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo(
+            "https://datastore.googleapis.com/v1/projects/MyCloudProject/operations/ASAzNjMwOTEyNjUJ");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("GET");
+    assertThat(httpRequest.getContent()).isNull();
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  @Test
+  public void testListOperations_all() throws IOException {
+    DatastoreAdmin.ListOperations listOperations = datastoreAdmin.listAll();
+    HttpRequest httpRequest = listOperations.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo("https://datastore.googleapis.com/v1/projects/MyCloudProject/operations");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("GET");
+    assertThat(httpRequest.getContent()).isNull();
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  @Test
+  public void testListOperations_filterByStartTime() throws IOException {
+    DatastoreAdmin.ListOperations listOperations =
+        datastoreAdmin.list("metadata.common.startTime>\"2018-10-31T00:00:00.0Z\"");
+    HttpRequest httpRequest = listOperations.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo(
+            "https://datastore.googleapis.com/v1/projects/MyCloudProject/operations"
+                + "?filter=metadata.common.startTime%3E%222018-10-31T00:00:00.0Z%22");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("GET");
+    assertThat(httpRequest.getContent()).isNull();
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  private static HttpRequest simulateSendRequest(HttpRequest httpRequest) {
+    try {
+      httpRequest.setUrl(new GenericUrl("https://localhost:65537")).execute();
+    } catch (Exception expected) {
+    }
+    return httpRequest;
+  }
+
+  private static Optional<String> getAccessToken(HttpRequest httpRequest) {
+    return httpRequest.getHeaders().getAuthorizationAsList().stream()
+        .filter(header -> header.startsWith(AUTH_HEADER_PREFIX))
+        .map(header -> header.substring(AUTH_HEADER_PREFIX.length()))
+        .findAny();
+  }
+
+  private static Optional<String> getRequestContent(HttpRequest httpRequest) throws IOException {
+    if (httpRequest.getContent() == null) {
+      return Optional.empty();
+    }
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    httpRequest.getContent().writeTo(outputStream);
+    outputStream.close();
+    return Optional.of(outputStream.toString(StandardCharsets.UTF_8.name()));
+  }
+}

javatests/google/registry/export/datastore/EntityFilterTest.java

@@ -0,0 +1,80 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.JUnitBackports.assertThrows;
+
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.common.collect.ImmutableList;
+import google.registry.testing.TestDataHelper;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for the instantiation, marshalling and unmarshalling of {@link EntityFilter}. */
+@RunWith(JUnit4.class)
+public class EntityFilterTest {
+
+  private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
+
+  @Test
+  public void testEntityFilter_create_nullKinds() {
+    assertThrows(NullPointerException.class, () -> new EntityFilter(null));
+  }
+
+  @Test
+  public void testEntityFilter_create_emptyKinds() {
+    assertThrows(IllegalArgumentException.class, () -> new EntityFilter(ImmutableList.of()));
+  }
+
+  @Test
+  public void testEntityFilter_marshall() throws IOException {
+    EntityFilter entityFilter =
+        new EntityFilter(ImmutableList.of("Registry", "Registrar", "DomainBase"));
+    assertThat(JSON_FACTORY.toString(entityFilter))
+        .isEqualTo(loadJsonString("entity_filter.json").replaceAll("[\\s\\n]+", ""));
+  }
+
+  @Test
+  public void testEntityFilter_unmarshall() throws IOException {
+    EntityFilter entityFilter = loadJson("entity_filter.json", EntityFilter.class);
+    assertThat(entityFilter.getKinds())
+        .containsExactly("Registry", "Registrar", "DomainBase")
+        .inOrder();
+  }
+
+  @Test
+  public void testEntityFilter_unmarshall_noKinds() throws IOException {
+    EntityFilter entityFilter = JSON_FACTORY.fromString("{}", EntityFilter.class);
+    assertThat(entityFilter.getKinds()).isEmpty();
+  }
+
+  @Test
+  public void testEntityFilter_unmarshall_emptyKinds() throws IOException {
+    EntityFilter entityFilter = JSON_FACTORY.fromString("{ \"kinds\" : [] }", EntityFilter.class);
+    assertThat(entityFilter.getKinds()).isEmpty();
+  }
+
+  private static <T> T loadJson(String fileName, Class<T> type) throws IOException {
+    return JSON_FACTORY.fromString(loadJsonString(fileName), type);
+  }
+
+  private static String loadJsonString(String fileName) {
+    return TestDataHelper.loadFile(EntityFilterTest.class, fileName);
+  }
+}

javatests/google/registry/export/datastore/OperationTest.java

@@ -0,0 +1,76 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import google.registry.export.datastore.Operation.CommonMetadata;
+import google.registry.export.datastore.Operation.Metadata;
+import google.registry.export.datastore.Operation.Progress;
+import google.registry.testing.TestDataHelper;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for unmarshalling {@link Operation} and its member types. */
+@RunWith(JUnit4.class)
+public class OperationTest {
+  private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
+
+  @Test
+  public void testCommonMetadata_unmarshall() throws IOException {
+    CommonMetadata commonMetadata = loadJson("common_metadata.json", CommonMetadata.class);
+    assertThat(commonMetadata.getState()).isEqualTo("SUCCESSFUL");
+    assertThat(commonMetadata.getOperationType()).isEqualTo("EXPORT_ENTITIES");
+  }
+
+  @Test
+  public void testProgress_unmarshall() throws IOException {
+    Progress progress = loadJson("progress.json", Progress.class);
+    assertThat(progress.getWorkCompleted()).isEqualTo(51797);
+    assertThat(progress.getWorkEstimated()).isEqualTo(54513);
+  }
+
+  @Test
+  public void testMetadata_unmarshall() throws IOException {
+    Metadata metadata = loadJson("metadata.json", Metadata.class);
+    assertThat(metadata.getCommonMetadata().getOperationType()).isEqualTo("EXPORT_ENTITIES");
+    assertThat(metadata.getCommonMetadata().getState()).isEqualTo("SUCCESSFUL");
+  }
+
+  @Test
+  public void testOperation_unmarshall() throws IOException {
+    Operation operation = loadJson("operation.json", Operation.class);
+    assertThat(operation.getName())
+        .startsWith("projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ");
+    assertThat(operation.isProcessing()).isTrue();
+    assertThat(operation.isSuccessful()).isFalse();
+    assertThat(operation.isDone()).isFalse();
+  }
+
+  @Test
+  public void testOperationList_unmarshall() throws IOException {
+    Operation.OperationList operationList =
+        loadJson("operation_list.json", Operation.OperationList.class);
+    assertThat(operationList.toList()).hasSize(2);
+  }
+
+  private static <T> T loadJson(String fileName, Class<T> type) throws IOException {
+    return JSON_FACTORY.fromString(TestDataHelper.loadFile(OperationTest.class, fileName), type);
+  }
+}

javatests/google/registry/export/datastore/testdata/common_metadata.json

@@ -0,0 +1,6 @@
+{
+    "startTime": "2018-10-29T16:01:04.645299Z",
+    "endTime": "2018-10-29T16:02:19.009859Z",
+    "operationType": "EXPORT_ENTITIES",
+    "state": "SUCCESSFUL"
+}

javatests/google/registry/export/datastore/testdata/entity_filter.json

@@ -0,0 +1,7 @@
+{
+  "kinds": [
+    "Registry",
+    "Registrar",
+    "DomainBase"
+  ]
+}

javatests/google/registry/export/datastore/testdata/export_request_content.json

@@ -0,0 +1,6 @@
+{
+  "entityFilter": {
+    "kinds": ["Registry", "Registrar", "DomainBase"]
+  },
+  "outputUrlPrefix": "gs://mybucket/path"
+}

javatests/google/registry/export/datastore/testdata/metadata.json

@@ -0,0 +1,25 @@
+{
+  "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+  "common": {
+    "startTime": "2018-10-29T16:01:04.645299Z",
+    "endTime": "2018-10-29T16:02:19.009859Z",
+    "operationType": "EXPORT_ENTITIES",
+    "state": "SUCCESSFUL"
+  },
+  "progressEntities": {
+    "workCompleted": "51797",
+    "workEstimated": "54513"
+  },
+  "progressBytes": {
+    "workCompleted": "96908367",
+    "workEstimated": "73773755"
+  },
+  "entityFilter": {
+    "kinds": [
+      "Registry",
+      "Registrar",
+      "DomainBase"
+    ]
+  },
+  "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+}

javatests/google/registry/export/datastore/testdata/operation.json

@@ -0,0 +1,19 @@
+{
+  "name": "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ",
+  "metadata": {
+    "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+    "common": {
+      "startTime": "2018-10-29T16:01:04.645299Z",
+      "operationType": "EXPORT_ENTITIES",
+      "state": "PROCESSING"
+    },
+    "entityFilter": {
+      "kinds": [
+        "Registry",
+        "Registrar",
+        "DomainBase"
+      ]
+    },
+    "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+  }
+}

javatests/google/registry/export/datastore/testdata/operation_list.json

@@ -0,0 +1,42 @@
+{
+  "operations": [
+    {
+      "name": "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ",
+      "metadata": {
+        "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+        "common": {
+          "startTime": "2018-10-29T16:01:04.645299Z",
+          "operationType": "EXPORT_ENTITIES",
+          "state": "PROCESSING"
+        },
+        "entityFilter": {
+          "kinds": [
+            "Registry",
+            "Registrar",
+            "DomainBase"
+          ]
+        },
+        "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+      }
+    },
+    {
+      "name": "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ",
+      "metadata": {
+        "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+        "common": {
+          "startTime": "2018-10-29T16:01:04.645299Z",
+          "operationType": "EXPORT_ENTITIES",
+          "state": "PROCESSING"
+        },
+        "entityFilter": {
+          "kinds": [
+            "Registry",
+            "Registrar",
+            "DomainBase"
+          ]
+        },
+        "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+      }
+    }
+  ]
+}

javatests/google/registry/export/datastore/testdata/progress.json

@@ -0,0 +1,4 @@
+{
+  "workCompleted": "51797",
+  "workEstimated": "54513"
+}