Add a package to open source

Add {java,tests}/google/registry/export/datastore to open source.
This is part of the migration to Datastore Managed Import/Export
for backup.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=221800709

javatests/google/registry/export/datastore/BUILD

@@ -0,0 +1,35 @@
+package(
+    default_testonly = 1,
+    default_visibility = ["//java/google/registry:registry_project"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//java/com/google/testing/builddefs:GenTestRules.bzl", "GenTestRules")
+
+java_library(
+    name = "datastore",
+    srcs = glob(["*.java"]),
+    resources = glob(["**/testdata/*.json"]),
+    deps = [
+        "//java/google/registry/export/datastore",
+        "//javatests/google/registry/testing",
+        "//third_party/java/google_http_java_client:http",
+        "//third_party/java/google_http_java_client:javanet",
+        "//third_party/java/google_http_java_client:util",
+        "@com_google_api_client",
+        "@com_google_guava",
+        "@com_google_http_client",
+        "@com_google_http_client_jackson2",
+        "@com_google_truth",
+        "@com_google_truth_extensions_truth_java8_extension",
+        "@junit",
+        "@org_mockito_all",
+    ],
+)
+
+GenTestRules(
+    name = "GeneratedTestRules",
+    test_files = glob(["*Test.java"]),
+    deps = [":datastore"],
+)

javatests/google/registry/export/datastore/DatastoreAdminTest.java

@@ -0,0 +1,161 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.common.collect.ImmutableList;
+import google.registry.testing.MockitoJUnitRule;
+import google.registry.testing.TestDataHelper;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link DatastoreAdmin}. */
+@RunWith(JUnit4.class)
+public class DatastoreAdminTest {
+
+  private static final String AUTH_HEADER_PREFIX = "Bearer ";
+  private static final String ACCESS_TOKEN = "MyAccessToken";
+  private static final ImmutableList<String> KINDS =
+      ImmutableList.of("Registry", "Registrar", "DomainBase");
+
+  @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create();
+
+  private HttpTransport httpTransport;
+  private GoogleCredential googleCredential;
+  private DatastoreAdmin datastoreAdmin;
+
+  @Before
+  public void setup() {
+    httpTransport = new NetHttpTransport();
+    googleCredential =
+        new GoogleCredential.Builder()
+            .setTransport(httpTransport)
+            .setJsonFactory(JacksonFactory.getDefaultInstance())
+            .setClock(() -> 0)
+            .build();
+    googleCredential.setAccessToken(ACCESS_TOKEN);
+    googleCredential.setExpiresInSeconds(1_000L);
+
+    datastoreAdmin =
+        new DatastoreAdmin.Builder(
+                googleCredential.getTransport(),
+                googleCredential.getJsonFactory(),
+                googleCredential)
+            .setApplicationName("MyApplication")
+            .setProjectId("MyCloudProject")
+            .build();
+  }
+
+  @Test
+  public void testExport() throws IOException {
+    DatastoreAdmin.Export export = datastoreAdmin.export("gs://mybucket/path", KINDS);
+    HttpRequest httpRequest = export.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo("https://datastore.googleapis.com/v1/projects/MyCloudProject:export");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("POST");
+
+    assertThat(getRequestContent(httpRequest))
+        .hasValue(
+            TestDataHelper.loadFile(getClass(), "export_request_content.json")
+                .replaceAll("[\\s\\n]+", ""));
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  @Test
+  public void testGetOperation() throws IOException {
+    DatastoreAdmin.Get get =
+        datastoreAdmin.get("projects/MyCloudProject/operations/ASAzNjMwOTEyNjUJ");
+    HttpRequest httpRequest = get.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo(
+            "https://datastore.googleapis.com/v1/projects/MyCloudProject/operations/ASAzNjMwOTEyNjUJ");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("GET");
+    assertThat(httpRequest.getContent()).isNull();
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  @Test
+  public void testListOperations_all() throws IOException {
+    DatastoreAdmin.ListOperations listOperations = datastoreAdmin.listAll();
+    HttpRequest httpRequest = listOperations.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo("https://datastore.googleapis.com/v1/projects/MyCloudProject/operations");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("GET");
+    assertThat(httpRequest.getContent()).isNull();
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  @Test
+  public void testListOperations_filterByStartTime() throws IOException {
+    DatastoreAdmin.ListOperations listOperations =
+        datastoreAdmin.list("metadata.common.startTime>\"2018-10-31T00:00:00.0Z\"");
+    HttpRequest httpRequest = listOperations.buildHttpRequest();
+    assertThat(httpRequest.getUrl().toString())
+        .isEqualTo(
+            "https://datastore.googleapis.com/v1/projects/MyCloudProject/operations"
+                + "?filter=metadata.common.startTime%3E%222018-10-31T00:00:00.0Z%22");
+    assertThat(httpRequest.getRequestMethod()).isEqualTo("GET");
+    assertThat(httpRequest.getContent()).isNull();
+
+    simulateSendRequest(httpRequest);
+    assertThat(getAccessToken(httpRequest)).hasValue(ACCESS_TOKEN);
+  }
+
+  private static HttpRequest simulateSendRequest(HttpRequest httpRequest) {
+    try {
+      httpRequest.setUrl(new GenericUrl("https://localhost:65537")).execute();
+    } catch (Exception expected) {
+    }
+    return httpRequest;
+  }
+
+  private static Optional<String> getAccessToken(HttpRequest httpRequest) {
+    return httpRequest.getHeaders().getAuthorizationAsList().stream()
+        .filter(header -> header.startsWith(AUTH_HEADER_PREFIX))
+        .map(header -> header.substring(AUTH_HEADER_PREFIX.length()))
+        .findAny();
+  }
+
+  private static Optional<String> getRequestContent(HttpRequest httpRequest) throws IOException {
+    if (httpRequest.getContent() == null) {
+      return Optional.empty();
+    }
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    httpRequest.getContent().writeTo(outputStream);
+    outputStream.close();
+    return Optional.of(outputStream.toString(StandardCharsets.UTF_8.name()));
+  }
+}

javatests/google/registry/export/datastore/EntityFilterTest.java

@@ -0,0 +1,80 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.JUnitBackports.assertThrows;
+
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.common.collect.ImmutableList;
+import google.registry.testing.TestDataHelper;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for the instantiation, marshalling and unmarshalling of {@link EntityFilter}. */
+@RunWith(JUnit4.class)
+public class EntityFilterTest {
+
+  private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
+
+  @Test
+  public void testEntityFilter_create_nullKinds() {
+    assertThrows(NullPointerException.class, () -> new EntityFilter(null));
+  }
+
+  @Test
+  public void testEntityFilter_create_emptyKinds() {
+    assertThrows(IllegalArgumentException.class, () -> new EntityFilter(ImmutableList.of()));
+  }
+
+  @Test
+  public void testEntityFilter_marshall() throws IOException {
+    EntityFilter entityFilter =
+        new EntityFilter(ImmutableList.of("Registry", "Registrar", "DomainBase"));
+    assertThat(JSON_FACTORY.toString(entityFilter))
+        .isEqualTo(loadJsonString("entity_filter.json").replaceAll("[\\s\\n]+", ""));
+  }
+
+  @Test
+  public void testEntityFilter_unmarshall() throws IOException {
+    EntityFilter entityFilter = loadJson("entity_filter.json", EntityFilter.class);
+    assertThat(entityFilter.getKinds())
+        .containsExactly("Registry", "Registrar", "DomainBase")
+        .inOrder();
+  }
+
+  @Test
+  public void testEntityFilter_unmarshall_noKinds() throws IOException {
+    EntityFilter entityFilter = JSON_FACTORY.fromString("{}", EntityFilter.class);
+    assertThat(entityFilter.getKinds()).isEmpty();
+  }
+
+  @Test
+  public void testEntityFilter_unmarshall_emptyKinds() throws IOException {
+    EntityFilter entityFilter = JSON_FACTORY.fromString("{ \"kinds\" : [] }", EntityFilter.class);
+    assertThat(entityFilter.getKinds()).isEmpty();
+  }
+
+  private static <T> T loadJson(String fileName, Class<T> type) throws IOException {
+    return JSON_FACTORY.fromString(loadJsonString(fileName), type);
+  }
+
+  private static String loadJsonString(String fileName) {
+    return TestDataHelper.loadFile(EntityFilterTest.class, fileName);
+  }
+}

javatests/google/registry/export/datastore/OperationTest.java

@@ -0,0 +1,76 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export.datastore;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import google.registry.export.datastore.Operation.CommonMetadata;
+import google.registry.export.datastore.Operation.Metadata;
+import google.registry.export.datastore.Operation.Progress;
+import google.registry.testing.TestDataHelper;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for unmarshalling {@link Operation} and its member types. */
+@RunWith(JUnit4.class)
+public class OperationTest {
+  private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
+
+  @Test
+  public void testCommonMetadata_unmarshall() throws IOException {
+    CommonMetadata commonMetadata = loadJson("common_metadata.json", CommonMetadata.class);
+    assertThat(commonMetadata.getState()).isEqualTo("SUCCESSFUL");
+    assertThat(commonMetadata.getOperationType()).isEqualTo("EXPORT_ENTITIES");
+  }
+
+  @Test
+  public void testProgress_unmarshall() throws IOException {
+    Progress progress = loadJson("progress.json", Progress.class);
+    assertThat(progress.getWorkCompleted()).isEqualTo(51797);
+    assertThat(progress.getWorkEstimated()).isEqualTo(54513);
+  }
+
+  @Test
+  public void testMetadata_unmarshall() throws IOException {
+    Metadata metadata = loadJson("metadata.json", Metadata.class);
+    assertThat(metadata.getCommonMetadata().getOperationType()).isEqualTo("EXPORT_ENTITIES");
+    assertThat(metadata.getCommonMetadata().getState()).isEqualTo("SUCCESSFUL");
+  }
+
+  @Test
+  public void testOperation_unmarshall() throws IOException {
+    Operation operation = loadJson("operation.json", Operation.class);
+    assertThat(operation.getName())
+        .startsWith("projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ");
+    assertThat(operation.isProcessing()).isTrue();
+    assertThat(operation.isSuccessful()).isFalse();
+    assertThat(operation.isDone()).isFalse();
+  }
+
+  @Test
+  public void testOperationList_unmarshall() throws IOException {
+    Operation.OperationList operationList =
+        loadJson("operation_list.json", Operation.OperationList.class);
+    assertThat(operationList.toList()).hasSize(2);
+  }
+
+  private static <T> T loadJson(String fileName, Class<T> type) throws IOException {
+    return JSON_FACTORY.fromString(TestDataHelper.loadFile(OperationTest.class, fileName), type);
+  }
+}

javatests/google/registry/export/datastore/testdata/common_metadata.json

@@ -0,0 +1,6 @@
+{
+    "startTime": "2018-10-29T16:01:04.645299Z",
+    "endTime": "2018-10-29T16:02:19.009859Z",
+    "operationType": "EXPORT_ENTITIES",
+    "state": "SUCCESSFUL"
+}

javatests/google/registry/export/datastore/testdata/entity_filter.json

@@ -0,0 +1,7 @@
+{
+  "kinds": [
+    "Registry",
+    "Registrar",
+    "DomainBase"
+  ]
+}

javatests/google/registry/export/datastore/testdata/export_request_content.json

@@ -0,0 +1,6 @@
+{
+  "entityFilter": {
+    "kinds": ["Registry", "Registrar", "DomainBase"]
+  },
+  "outputUrlPrefix": "gs://mybucket/path"
+}

javatests/google/registry/export/datastore/testdata/metadata.json

@@ -0,0 +1,25 @@
+{
+  "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+  "common": {
+    "startTime": "2018-10-29T16:01:04.645299Z",
+    "endTime": "2018-10-29T16:02:19.009859Z",
+    "operationType": "EXPORT_ENTITIES",
+    "state": "SUCCESSFUL"
+  },
+  "progressEntities": {
+    "workCompleted": "51797",
+    "workEstimated": "54513"
+  },
+  "progressBytes": {
+    "workCompleted": "96908367",
+    "workEstimated": "73773755"
+  },
+  "entityFilter": {
+    "kinds": [
+      "Registry",
+      "Registrar",
+      "DomainBase"
+    ]
+  },
+  "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+}

javatests/google/registry/export/datastore/testdata/operation.json

@@ -0,0 +1,19 @@
+{
+  "name": "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ",
+  "metadata": {
+    "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+    "common": {
+      "startTime": "2018-10-29T16:01:04.645299Z",
+      "operationType": "EXPORT_ENTITIES",
+      "state": "PROCESSING"
+    },
+    "entityFilter": {
+      "kinds": [
+        "Registry",
+        "Registrar",
+        "DomainBase"
+      ]
+    },
+    "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+  }
+}

javatests/google/registry/export/datastore/testdata/operation_list.json

@@ -0,0 +1,42 @@
+{
+  "operations": [
+    {
+      "name": "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ",
+      "metadata": {
+        "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+        "common": {
+          "startTime": "2018-10-29T16:01:04.645299Z",
+          "operationType": "EXPORT_ENTITIES",
+          "state": "PROCESSING"
+        },
+        "entityFilter": {
+          "kinds": [
+            "Registry",
+            "Registrar",
+            "DomainBase"
+          ]
+        },
+        "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+      }
+    },
+    {
+      "name": "projects/domain-registry-alpha/operations/ASAzNjMwOTEyNjUJ",
+      "metadata": {
+        "@type": "type.googleapis.com/google.datastore.admin.v1.ExportEntitiesMetadata",
+        "common": {
+          "startTime": "2018-10-29T16:01:04.645299Z",
+          "operationType": "EXPORT_ENTITIES",
+          "state": "PROCESSING"
+        },
+        "entityFilter": {
+          "kinds": [
+            "Registry",
+            "Registrar",
+            "DomainBase"
+          ]
+        },
+        "outputUrlPrefix": "gs://domain-registry-alpha-datastore-export-test/2018-10-29T16:01:04_99364"
+      }
+    }
+  ]
+}

javatests/google/registry/export/datastore/testdata/progress.json

@@ -0,0 +1,4 @@
+{
+  "workCompleted": "51797",
+  "workEstimated": "54513"
+}