Add a Secret Manager client for Nomulus (#872)

* Add a Secret Manager client for Nomulus

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClient.java

@@ -0,0 +1,126 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.secretmanager.v1.SecretVersion;
+import com.google.common.collect.Streams;
+import java.util.Optional;
+
+/** A Cloud Secret Manager client for Nomulus, bound to a specific GCP project. */
+public interface SecretManagerClient {
+
+  /**
+   * Creates a new secret in the Cloud Secret Manager with no data.
+   *
+   * <p>Use addVersion to add data to this secret.
+   *
+   * @param secretId The ID of the secret, must be unique in a project
+   * @throws SecretAlreadyExistsException A secret with this secretId already exists
+   */
+  void createSecret(String secretId);
+
+  /** Returns all secret IDs in the Cloud Secret Manager. */
+  Iterable<String> listSecrets();
+
+  /** Returns the {@link SecretVersionState} of all secrets with {@code secretId}. */
+  Iterable<SecretVersionState> listSecretVersions(String secretId);
+
+  /**
+   * Returns the version strings of all secrets in the given {@code state} with {@code secretId}.
+   */
+  default Iterable<String> listSecretVersions(String secretId, SecretVersion.State state) {
+    return () ->
+        Streams.stream(listSecretVersions(secretId))
+            .filter(secretVersionState -> secretVersionState.state().equals(state))
+            .map(SecretVersionState::version)
+            .iterator();
+  }
+
+  /**
+   * Adds a new version of data to a secret.
+   *
+   * @param secretId The ID of the secret
+   * @param data The secret data to be stored in Cloud Secret Manager, encoded in utf-8 charset
+   * @return The version string of the newly added secret data
+   */
+  String addSecretVersion(String secretId, String data);
+
+  /**
+   * Returns the data of a secret at the given version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  String getSecretData(String secretId, Optional<String> version);
+
+  /**
+   * Destroys a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to destroy
+   */
+  void destroySecretVersion(String secretId, String version);
+
+  /**
+   * Deletes a secret from the Secret Manager. All versions of this secret will be destroyed.
+   *
+   * @param secretId The ID of the secret to be deleted
+   */
+  void deleteSecret(String secretId);
+
+  /** Contains the {@link SecretVersion.State State} of an secret version. */
+  @AutoValue
+  abstract class SecretVersionState {
+
+    public abstract String secretId();
+
+    public abstract String version();
+
+    public abstract SecretVersion.State state();
+
+    public static SecretVersionState of(
+        String secretId, String version, SecretVersion.State state) {
+      return new AutoValue_SecretManagerClient_SecretVersionState(secretId, version, state);
+    }
+  }
+
+  /** Catch-all class for all SecretManager exceptions. */
+  class SecretManagerException extends RuntimeException {
+    SecretManagerException(Throwable cause) {
+      super(cause);
+    }
+  }
+
+  /** The secret to be created already exists. */
+  class SecretAlreadyExistsException extends SecretManagerException {
+    SecretAlreadyExistsException(Throwable cause) {
+      super(cause);
+    }
+  }
+
+  /**
+   * The resource being requested in the Secret Manager does not exist.
+   *
+   * <p>The missing resource may be a secret version or the secret itself. They are grouped together
+   * because it is not always possible to identify the type of the missing resource.
+   */
+  class NoSuchSecretResourceException extends SecretManagerException {
+    NoSuchSecretResourceException(Throwable cause) {
+      super(cause);
+    }
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClientImpl.java

@@ -0,0 +1,151 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.api.gax.rpc.ApiException;
+import com.google.cloud.secretmanager.v1.ProjectName;
+import com.google.cloud.secretmanager.v1.Replication;
+import com.google.cloud.secretmanager.v1.Replication.Automatic;
+import com.google.cloud.secretmanager.v1.Secret;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretVersionsPagedResponse;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceClient.ListSecretsPagedResponse;
+import com.google.cloud.secretmanager.v1.SecretName;
+import com.google.cloud.secretmanager.v1.SecretPayload;
+import com.google.cloud.secretmanager.v1.SecretVersion;
+import com.google.cloud.secretmanager.v1.SecretVersionName;
+import com.google.common.collect.Iterables;
+import com.google.protobuf.ByteString;
+import google.registry.util.Retrier;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import javax.inject.Inject;
+
+/** Implements {@link SecretManagerClient} on Google Cloud Platform. */
+public class SecretManagerClientImpl implements SecretManagerClient {
+  private final String project;
+  private final SecretManagerServiceClient csmClient;
+  private final Retrier retrier;
+
+  @Inject
+  SecretManagerClientImpl(String project, SecretManagerServiceClient csmClient, Retrier retrier) {
+    this.project = project;
+    this.csmClient = csmClient;
+    this.retrier = retrier;
+  }
+
+  @Override
+  public void createSecret(String secretId) {
+    checkNotNull(secretId, "secretId");
+    Secret secretSettings = Secret.newBuilder().setReplication(defaultReplicationPolicy()).build();
+    callSecretManager(
+        () -> csmClient.createSecret(ProjectName.of(project), secretId, secretSettings));
+  }
+
+  @Override
+  public Iterable<String> listSecrets() {
+    ListSecretsPagedResponse response =
+        callSecretManager(() -> csmClient.listSecrets(ProjectName.of(project)));
+    return Iterables.transform(
+        response.iterateAll(), secret -> SecretName.parse(secret.getName()).getSecret());
+  }
+
+  @Override
+  public Iterable<SecretVersionState> listSecretVersions(String secretId) {
+    checkNotNull(secretId, "secretId");
+    ListSecretVersionsPagedResponse response =
+        callSecretManager(() -> csmClient.listSecretVersions(SecretName.of(project, secretId)));
+    return Iterables.transform(
+        response.iterateAll(), SecretManagerClientImpl::toSecretVersionState);
+  }
+
+  private static SecretVersionState toSecretVersionState(SecretVersion secretVersion) {
+    SecretVersionName name = SecretVersionName.parse(secretVersion.getName());
+    return SecretVersionState.of(
+        name.getSecret(), name.getSecretVersion(), secretVersion.getState());
+  }
+
+  @Override
+  public String addSecretVersion(String secretId, String data) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(data, "data");
+    SecretName secretName = SecretName.of(project, secretId);
+    SecretPayload secretPayload =
+        SecretPayload.newBuilder().setData(ByteString.copyFromUtf8(data)).build();
+    SecretVersion response =
+        callSecretManager(() -> csmClient.addSecretVersion(secretName, secretPayload));
+    checkState(SecretVersionName.isParsableFrom(response.getName()));
+    SecretVersionName secretVersionName = SecretVersionName.parse(response.getName());
+    return secretVersionName.getSecretVersion();
+  }
+
+  @Override
+  public String getSecretData(String secretId, Optional<String> version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    return callSecretManager(
+        () ->
+            csmClient
+                .accessSecretVersion(
+                    SecretVersionName.of(project, secretId, version.orElse("latest")))
+                .getPayload()
+                .getData()
+                .toStringUtf8());
+  }
+
+  @Override
+  public void destroySecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.destroySecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
+  @Override
+  public void deleteSecret(String secretId) {
+    checkNotNull(secretId, "secretId");
+    callSecretManager(
+        () -> {
+          csmClient.deleteSecret(SecretName.of(project, secretId));
+          return null;
+        });
+  }
+
+  private <T> T callSecretManager(Callable<T> callable) {
+    try {
+      return retrier.callWithRetry(callable, SecretManagerClientImpl::isRetryableException);
+    } catch (ApiException e) {
+      switch (e.getStatusCode().getCode()) {
+        case ALREADY_EXISTS:
+          throw new SecretAlreadyExistsException(e);
+        case NOT_FOUND:
+          throw new NoSuchSecretResourceException(e);
+        default:
+          throw new SecretManagerException(e);
+      }
+    }
+  }
+
+  private static boolean isRetryableException(Throwable e) {
+    return e instanceof ApiException && ((ApiException) e).isRetryable();
+  }
+
+  private static Replication defaultReplicationPolicy() {
+    return Replication.newBuilder().setAutomatic(Automatic.newBuilder().build()).build();
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerModule.java

@@ -0,0 +1,54 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
+import dagger.Component;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.util.Retrier;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import javax.inject.Singleton;
+
+/** Provides bindings for {@link SecretManagerClient}. */
+@Module
+public class SecretManagerModule {
+
+  private final String project;
+
+  public SecretManagerModule(String project) {
+    this.project = checkNotNull(project, "project");
+  }
+
+  @Provides
+  @Singleton
+  SecretManagerClient provideSecretManagerClient(Retrier retrier) {
+    try {
+      return new SecretManagerClientImpl(project, SecretManagerServiceClient.create(), retrier);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Singleton
+  @Component(modules = {ConfigModule.class, SecretManagerModule.class, UtilsModule.class})
+  public interface SecretManagerComponent {
+    SecretManagerClient secretManagerClient();
+  }
+}

core/src/test/java/google/registry/privileges/secretmanager/FakeSecretManagerClient.java

@@ -0,0 +1,170 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.cloud.secretmanager.v1.SecretVersion.State;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Optional;
+import javax.inject.Inject;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableList;
+
+/** Implements {@link SecretManagerClient} for tests. */
+public class FakeSecretManagerClient implements SecretManagerClient {
+
+  private final HashMap<String, SecretEntry> secrets = new HashMap<>();
+
+  @Inject
+  FakeSecretManagerClient() {}
+
+  @Override
+  public void createSecret(String secretId) {
+    checkNotNull(secretId, "secretId");
+    if (secrets.containsKey(secretId)) {
+      throw new SecretAlreadyExistsException(null);
+    }
+    secrets.put(secretId, new SecretEntry(secretId));
+  }
+
+  @Override
+  public Iterable<String> listSecrets() {
+    return ImmutableSet.copyOf(secrets.keySet());
+  }
+
+  @Override
+  public Iterable<SecretVersionState> listSecretVersions(String secretId) {
+    checkNotNull(secretId, "secretId");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    return secretEntry.listVersions();
+  }
+
+  @Override
+  public String addSecretVersion(String secretId, String data) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(data, "data");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    return secretEntry.addVersion(data);
+  }
+
+  @Override
+  public String getSecretData(String secretId, Optional<String> version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    return secretEntry.getVersion(version).getData();
+  }
+
+  @Override
+  public void destroySecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    secretEntry.destroyVersion(version);
+  }
+
+  @Override
+  public void deleteSecret(String secretId) {
+    checkNotNull(secretId, "secretId");
+    if (!secrets.containsKey(secretId)) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    secrets.remove(secretId);
+  }
+
+  private static class VersionEntry {
+    private String data;
+    private State state;
+
+    VersionEntry(String data) {
+      this.data = checkNotNull(data, "data");
+      this.state = State.ENABLED;
+    }
+
+    String getData() {
+      if (state != State.ENABLED) {
+        throw new SecretManagerException(null);
+      }
+      return data;
+    }
+
+    State getState() {
+      return state;
+    }
+
+    void destroy() {
+      data = null;
+      state = State.DESTROYED;
+    }
+  }
+
+  private static class SecretEntry {
+    private final String secretId;
+    private ArrayList<VersionEntry> versions;
+
+    SecretEntry(String secretId) {
+      this.secretId = secretId;
+      versions = new ArrayList<>();
+    }
+
+    String addVersion(String data) {
+      VersionEntry versionEntry = new VersionEntry(data);
+      versions.add(versionEntry);
+      return String.valueOf(versions.size() - 1);
+    }
+
+    VersionEntry getVersion(Optional<String> version) {
+      try {
+        int index = version.map(Integer::valueOf).orElse(versions.size() - 1);
+        return versions.get(index);
+      } catch (NumberFormatException e) {
+        throw new IllegalArgumentException("Invalid version " + version.get());
+      }
+    }
+
+    Iterable<SecretVersionState> listVersions() {
+      ImmutableList.Builder<SecretVersionState> builder = new ImmutableList.Builder<>();
+      for (int i = 0; i < versions.size(); i++) {
+        builder.add(SecretVersionState.of(secretId, String.valueOf(i), versions.get(i).getState()));
+      }
+      return builder.build();
+    }
+
+    void destroyVersion(String version) {
+      try {
+        int index = Integer.valueOf(version);
+        versions.get(index).destroy();
+      } catch (NumberFormatException e) {
+        throw new IllegalArgumentException("Invalid version " + version);
+      } catch (ArrayIndexOutOfBoundsException e) {
+        throw new NoSuchSecretResourceException(null);
+      }
+    }
+  }
+}

core/src/test/java/google/registry/privileges/secretmanager/SecretManagerClientTest.java

@@ -0,0 +1,143 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.google.cloud.secretmanager.v1.SecretVersion.State;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
+import java.util.Optional;
+import java.util.UUID;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link SecretManagerClient}.
+ *
+ * <p>If the 'test.gcp_integration.env' system property is not set, this class serves as unit tests
+ * for {@link FakeSecretManagerClient}.
+ *
+ * <p>If the 'test.gcp_integration.env' environment variable is set, this class serves as
+ * integration tests with a GCP project whose name is specified by the variable.
+ *
+ * <p>See <a href="../../../../../../../../java_common.gradle">java_common.gradle</a> for more
+ * information.
+ */
+public class SecretManagerClientTest {
+  // Common prefix for all secret ids generated in this test.
+  private static final String SECRET_ID_PREFIX = "TEST_" + UUID.randomUUID() + "_";
+  // Used for unique secret id generation.
+  private static int seqno = 0;
+
+  private static SecretManagerClient secretManagerClient;
+  private static boolean isUnitTest = true;
+
+  private static String nextSecretId() {
+    return SECRET_ID_PREFIX + seqno++;
+  }
+
+  @BeforeAll
+  static void beforeAll() {
+    String environmentName = System.getProperty("test.gcp_integration.env");
+    if (environmentName != null) {
+      secretManagerClient =
+          DaggerSecretManagerModule_SecretManagerComponent.builder()
+              .secretManagerModule(
+                  new SecretManagerModule(String.format("domain-registry-%s", environmentName)))
+              .build()
+              .secretManagerClient();
+      isUnitTest = false;
+    } else {
+      secretManagerClient = new FakeSecretManagerClient();
+    }
+  }
+
+  @AfterAll
+  static void afterAll() {
+    if (isUnitTest) {
+      return;
+    }
+    for (String secretId : secretManagerClient.listSecrets()) {
+      if (secretId.startsWith(SECRET_ID_PREFIX)) {
+        secretManagerClient.deleteSecret(secretId);
+      }
+    }
+  }
+
+  @Test
+  void createSecret_success() {
+    String secretId = nextSecretId();
+    secretManagerClient.createSecret(secretId);
+    assertThat(secretManagerClient.listSecrets()).contains(secretId);
+  }
+
+  @Test
+  void createSecret_duplicate() {
+    String secretId = nextSecretId();
+    secretManagerClient.createSecret(secretId);
+    assertThrows(
+        SecretAlreadyExistsException.class, () -> secretManagerClient.createSecret(secretId));
+  }
+
+  @Test
+  void addSecretVersion() {
+    String secretId = nextSecretId();
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED))
+        .containsExactly(version);
+  }
+
+  @Test
+  void getSecretData_byVersion() {
+    String secretId = nextSecretId();
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    assertThat(secretManagerClient.getSecretData(secretId, Optional.of(version)))
+        .isEqualTo("mydata");
+  }
+
+  @Test
+  void getSecretData_latestVersion() {
+    String secretId = nextSecretId();
+    secretManagerClient.createSecret(secretId);
+    secretManagerClient.addSecretVersion(secretId, "mydata");
+    assertThat(secretManagerClient.getSecretData(secretId, Optional.empty())).isEqualTo("mydata");
+  }
+
+  @Test
+  void destroySecretVersion() {
+    String secretId = nextSecretId();
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.destroySecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DESTROYED)).contains(version);
+    assertThrows(
+        SecretManagerException.class,
+        () -> secretManagerClient.getSecretData(secretId, Optional.of(version)));
+  }
+
+  @Test
+  void deleteSecret() {
+    String secretId = nextSecretId();
+    secretManagerClient.createSecret(secretId);
+    assertThat(secretManagerClient.listSecrets()).contains(secretId);
+    secretManagerClient.deleteSecret(secretId);
+    assertThat(secretManagerClient.listSecrets()).doesNotContain(secretId);
+  }
+}