Add initial support to write/update reserved lists to Cloud SQL (#388)

* Add initial support to write reserved list to Cloud SQL

* Add support to update reserved list in Cloud SQL

* Fix wrong check when override is enabled in create command

* Add sql dependent tests to the suite

* Address comment

* Make invocation of super.execute more readable

* Add test to check upper case and non-puny code label

* Move ReservedListDao related classes to schema package

* Simplify a function name

core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java

@@ -26,7 +26,10 @@ import google.registry.persistence.UpdateAutoTimestampConverterTest;
 import google.registry.persistence.ZonedDateTimeConverterTest;
 import google.registry.schema.cursor.CursorDaoTest;
 import google.registry.schema.tld.PremiumListDaoTest;
+import google.registry.schema.tld.ReservedListDaoTest;
 import google.registry.schema.tmch.ClaimsListDaoTest;
+import google.registry.tools.CreateReservedListCommandTest;
+import google.registry.tools.UpdateReservedListCommandTest;
 import google.registry.ui.server.registrar.RegistryLockGetActionTest;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
@@ -47,6 +50,7 @@ import org.junit.runners.Suite.SuiteClasses;
   BloomFilterConverterTest.class,
   ClaimsListDaoTest.class,
   CreateAutoTimestampConverterTest.class,
+  CreateReservedListCommandTest.class,
   CurrencyUnitConverterTest.class,
   CursorDaoTest.class,
   DateTimeConverterTest.class,
@@ -56,7 +60,9 @@ import org.junit.runners.Suite.SuiteClasses;
   PremiumListDaoTest.class,
   RegistryLockDaoTest.class,
   RegistryLockGetActionTest.class,
+  ReservedListDaoTest.class,
   UpdateAutoTimestampConverterTest.class,
+  UpdateReservedListCommandTest.class,
   ZonedDateTimeConverterTest.class
 })
 public class SqlIntegrationTestSuite {}

core/src/test/java/google/registry/schema/tld/ReservedListDaoTest.java

@@ -0,0 +1,69 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.schema.tld;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.collect.ImmutableMap;
+import google.registry.model.registry.label.ReservationType;
+import google.registry.model.transaction.JpaTransactionManagerRule;
+import google.registry.schema.tld.ReservedList.ReservedEntry;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link ReservedListDao}. */
+@RunWith(JUnit4.class)
+public class ReservedListDaoTest {
+  @Rule
+  public final JpaTransactionManagerRule jpaTmRule =
+      new JpaTransactionManagerRule.Builder().build();
+
+  private static final ImmutableMap<String, ReservedEntry> TEST_RESERVATIONS =
+      ImmutableMap.of(
+          "food",
+          ReservedEntry.create(ReservationType.RESERVED_FOR_SPECIFIC_USE, null),
+          "music",
+          ReservedEntry.create(ReservationType.FULLY_BLOCKED, "fully blocked"));
+
+  @Test
+  public void save_worksSuccessfully() {
+    ReservedList reservedList = ReservedList.create("testname", false, TEST_RESERVATIONS);
+    ReservedListDao.save(reservedList);
+    jpaTm()
+        .transact(
+            () -> {
+              ReservedList persistedList =
+                  jpaTm()
+                      .getEntityManager()
+                      .createQuery("FROM ReservedList WHERE name = :name", ReservedList.class)
+                      .setParameter("name", "testname")
+                      .getSingleResult();
+              assertThat(persistedList.getLabelsToReservations())
+                  .containsExactlyEntriesIn(TEST_RESERVATIONS);
+              assertThat(persistedList.getCreationTimestamp())
+                  .isEqualTo(jpaTmRule.getTxnClock().nowUtc());
+            });
+  }
+
+  @Test
+  public void checkExists_worksSuccessfully() {
+    assertThat(ReservedListDao.checkExists("testlist")).isFalse();
+    ReservedListDao.save(ReservedList.create("testlist", false, TEST_RESERVATIONS));
+    assertThat(ReservedListDao.checkExists("testlist")).isTrue();
+  }
+}

core/src/test/java/google/registry/schema/tld/ReservedListTest.java

@@ -15,6 +15,8 @@
 package google.registry.schema.tld;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
+import static google.registry.testing.JUnitBackports.assertThrows;
 
 import com.google.common.collect.ImmutableMap;
 import google.registry.model.registry.label.ReservationType;
@@ -50,4 +52,34 @@ public class ReservedListTest {
             ReservedEntry.create(
                 ReservationType.RESERVED_FOR_ANCHOR_TENANT, "reserved for anchor tenant"));
   }
+
+  @Test
+  public void create_throwsExceptionWhenLabelIsNotLowercase() {
+    Exception e =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                ReservedList.create(
+                    "UPPER.tld",
+                    true,
+                    ImmutableMap.of("UPPER", ReservedEntry.create(FULLY_BLOCKED, ""))));
+    assertThat(e)
+        .hasMessageThat()
+        .contains("Label(s) [UPPER] must be in puny-coded, lower-case form");
+  }
+
+  @Test
+  public void create_labelMustBePunyCoded() {
+    Exception e =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                ReservedList.create(
+                    "lower.みんな",
+                    true,
+                    ImmutableMap.of("みんな", ReservedEntry.create(FULLY_BLOCKED, ""))));
+    assertThat(e)
+        .hasMessageThat()
+        .contains("Label(s) [みんな] must be in puny-coded, lower-case form");
+  }
 }

core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTest.java

@@ -0,0 +1,126 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.registry.label.ReservationType.ALLOWED_IN_SUNRISE;
+import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
+import static google.registry.testing.JUnitBackports.assertThrows;
+import static google.registry.tools.CreateOrUpdateReservedListCommand.parseToReservationsByLabels;
+
+import com.google.common.collect.ImmutableList;
+import google.registry.model.registry.label.ReservationType;
+import google.registry.schema.tld.ReservedList.ReservedEntry;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link CreateOrUpdateReservedListCommand}. */
+@RunWith(JUnit4.class)
+public class CreateOrUpdateReservedListCommandTest {
+
+  @Test
+  public void parseToReservationsByLabels_worksCorrectly() {
+    assertThat(
+            parseToReservationsByLabels(
+                ImmutableList.of(
+                    "reserveddomain,FULLY_BLOCKED",
+                    "availableinga,ALLOWED_IN_SUNRISE#allowed_in_sunrise",
+                    "fourletterword,FULLY_BLOCKED")))
+        .containsExactly(
+            "reserveddomain",
+            ReservedEntry.create(ReservationType.FULLY_BLOCKED, ""),
+            "availableinga",
+            ReservedEntry.create(ALLOWED_IN_SUNRISE, "allowed_in_sunrise"),
+            "fourletterword",
+            ReservedEntry.create(FULLY_BLOCKED, ""));
+  }
+
+  @Test
+  public void parseToReservationsByLabels_throwsExceptionForInvalidLabel() {
+    Throwable thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                parseToReservationsByLabels(
+                    ImmutableList.of("reserveddomain,FULLY_BLOCKED", "UPPER,FULLY_BLOCKED")));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("Label 'UPPER' must be in puny-coded, lower-case form");
+  }
+
+  @Test
+  public void parseToReservationsByLabels_throwsExceptionForNonPunyCodedLabel() {
+    Throwable thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                parseToReservationsByLabels(
+                    ImmutableList.of("reserveddomain,FULLY_BLOCKED", "みんな,FULLY_BLOCKED")));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("Label 'みんな' must be in puny-coded, lower-case form");
+  }
+
+  @Test
+  public void parseToReservationsByLabels_throwsExceptionForInvalidReservationType() {
+    Throwable thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                parseToReservationsByLabels(
+                    ImmutableList.of(
+                        "reserveddomain,FULLY_BLOCKED", "invalidtype,INVALID_RESERVATION_TYPE")));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "No enum constant"
+                + " google.registry.model.registry.label.ReservationType.INVALID_RESERVATION_TYPE");
+  }
+
+  @Test
+  public void parseToReservationsByLabels_throwsExceptionForInvalidLines() {
+    Throwable thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                parseToReservationsByLabels(
+                    ImmutableList.of(
+                        "reserveddomain,FULLY_BLOCKED,too,many,parts",
+                        "fourletterword,FULLY_BLOCKED")));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Could not parse line in reserved list: reserveddomain,FULLY_BLOCKED,too,many,parts");
+  }
+
+  @Test
+  public void parseToReservationsByLabels_throwsExceptionForDuplicateEntries() {
+    Throwable thrown =
+        assertThrows(
+            IllegalStateException.class,
+            () ->
+                parseToReservationsByLabels(
+                    ImmutableList.of(
+                        "reserveddomain,FULLY_BLOCKED",
+                        "fourletterword,FULLY_BLOCKED",
+                        "fourletterword,FULLY_BLOCKED")));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Reserved list cannot contain duplicate labels. Dupes (with counts) were:"
+                + " [fourletterword x 2]");
+  }
+}

core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTestCase.java

@@ -14,15 +14,26 @@
 
 package google.registry.tools;
 
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
+import static google.registry.model.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.testing.JUnitBackports.assertThrows;
 import static google.registry.testing.TestDataHelper.loadFile;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.beust.jcommander.ParameterException;
 import com.google.common.io.Files;
+import com.google.common.truth.Truth8;
+import google.registry.model.registry.label.ReservedList;
+import google.registry.model.transaction.JpaTransactionManagerRule;
+import google.registry.schema.tld.ReservedList.ReservedEntry;
+import google.registry.schema.tld.ReservedListDao;
 import java.io.File;
 import java.io.IOException;
+import java.util.Map;
+import javax.persistence.EntityManager;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 
 /**
@@ -33,6 +44,10 @@ import org.junit.Test;
 public abstract class CreateOrUpdateReservedListCommandTestCase
     <T extends CreateOrUpdateReservedListCommand> extends CommandTestCase<T> {
 
+  @Rule
+  public final JpaTransactionManagerRule jpaTmRule =
+      new JpaTransactionManagerRule.Builder().build();
+
   String reservedTermsPath;
   String invalidReservedTermsPath;
 
@@ -64,4 +79,52 @@ public abstract class CreateOrUpdateReservedListCommandTestCase
         IllegalArgumentException.class,
         () -> runCommandForced("--name=xn--q9jyb4c-blork", "--input=" + invalidReservedTermsPath));
   }
+
+  google.registry.schema.tld.ReservedList createCloudSqlReservedList(
+      String name, boolean shouldPublish, Map<String, ReservedEntry> labelsToEntries) {
+    return google.registry.schema.tld.ReservedList.create(name, shouldPublish, labelsToEntries);
+  }
+
+  google.registry.schema.tld.ReservedList getCloudSqlReservedList(String name) {
+    return jpaTm()
+        .transact(
+            () -> {
+              EntityManager em = jpaTm().getEntityManager();
+              long revisionId =
+                  em.createQuery(
+                          "SELECT MAX(rl.revisionId) FROM ReservedList rl WHERE name = :name",
+                          Long.class)
+                      .setParameter("name", name)
+                      .getSingleResult();
+              return em.createQuery(
+                      "FROM ReservedList rl LEFT JOIN FETCH rl.labelsToReservations WHERE"
+                          + " rl.revisionId = :revisionId",
+                      google.registry.schema.tld.ReservedList.class)
+                  .setParameter("revisionId", revisionId)
+                  .getSingleResult();
+            });
+  }
+
+  void verifyXnq9jyb4cInCloudSql() {
+    assertThat(ReservedListDao.checkExists("xn--q9jyb4c_common-reserved")).isTrue();
+    google.registry.schema.tld.ReservedList persistedList =
+        getCloudSqlReservedList("xn--q9jyb4c_common-reserved");
+    assertThat(persistedList.getName()).isEqualTo("xn--q9jyb4c_common-reserved");
+    assertThat(persistedList.getShouldPublish()).isTrue();
+    assertThat(persistedList.getCreationTimestamp()).isEqualTo(jpaTmRule.getTxnClock().nowUtc());
+    assertThat(persistedList.getLabelsToReservations())
+        .containsExactly(
+            "baddies",
+            ReservedEntry.create(FULLY_BLOCKED, ""),
+            "ford",
+            ReservedEntry.create(FULLY_BLOCKED, "random comment"));
+  }
+
+  void verifyXnq9jyb4cInDatastore() {
+    Truth8.assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent();
+    ReservedList reservedList = ReservedList.get("xn--q9jyb4c_common-reserved").get();
+    assertThat(reservedList.getReservedListEntries()).hasSize(2);
+    Truth8.assertThat(reservedList.getReservationInList("baddies")).hasValue(FULLY_BLOCKED);
+    Truth8.assertThat(reservedList.getReservationInList("ford")).hasValue(FULLY_BLOCKED);
+  }
 }

core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java

@@ -24,8 +24,11 @@ import static google.registry.testing.JUnitBackports.assertThrows;
 import static google.registry.tools.CreateReservedListCommand.INVALID_FORMAT_ERROR_MESSAGE;
 import static org.joda.time.DateTimeZone.UTC;
 
+import com.google.common.collect.ImmutableMap;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.ReservedList;
+import google.registry.schema.tld.ReservedList.ReservedEntry;
+import google.registry.schema.tld.ReservedListDao;
 import org.joda.time.DateTime;
 import org.junit.Before;
 import org.junit.Test;
@@ -173,6 +176,28 @@ public class CreateReservedListCommandTest extends
     runNameTestExpectedFailure("soy_$oy", INVALID_FORMAT_ERROR_MESSAGE);
   }
 
+  @Test
+  public void testSaveToCloudSql_succeeds() throws Exception {
+    runCommandForced(
+        "--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath, "--also_cloud_sql");
+    verifyXnq9jyb4cInDatastore();
+    verifyXnq9jyb4cInCloudSql();
+  }
+
+  @Test
+  public void testSaveToCloudSql_noExceptionThrownWhenSaveFail() throws Exception {
+    // Note that, during the dual-write phase, we want to make sure that no exception will be
+    // thrown if saving reserved list to Cloud SQL fails.
+    ReservedListDao.save(
+        createCloudSqlReservedList(
+            "xn--q9jyb4c_common-reserved",
+            true,
+            ImmutableMap.of("testdomain", ReservedEntry.create(FULLY_BLOCKED, ""))));
+    runCommandForced(
+        "--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath, "--also_cloud_sql");
+    verifyXnq9jyb4cInDatastore();
+  }
+
   private void runNameTestExpectedFailure(String name, String expectedErrorMsg) {
     IllegalArgumentException thrown =
         assertThrows(

core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java

@@ -22,14 +22,17 @@ import static google.registry.testing.JUnitBackports.assertThrows;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import google.registry.model.registry.label.ReservedList;
+import google.registry.schema.tld.ReservedList.ReservedEntry;
+import google.registry.schema.tld.ReservedListDao;
 import org.junit.Test;
 
 /** Unit tests for {@link UpdateReservedListCommand}. */
 public class UpdateReservedListCommandTest extends
     CreateOrUpdateReservedListCommandTestCase<UpdateReservedListCommand> {
 
-  private void populateInitialReservedList(boolean shouldPublish) {
+  private void populateInitialReservedListInDatastore(boolean shouldPublish) {
     persistResource(
         new ReservedList.Builder()
             .setName("xn--q9jyb4c_common-reserved")
@@ -40,6 +43,14 @@ public class UpdateReservedListCommandTest extends
             .build());
   }
 
+  private void populateInitialReservedListInCloudSql(boolean shouldPublish) {
+    ReservedListDao.save(
+        createCloudSqlReservedList(
+            "xn--q9jyb4c_common-reserved",
+            shouldPublish,
+            ImmutableMap.of("helicopter", ReservedEntry.create(FULLY_BLOCKED, ""))));
+  }
+
   @Test
   public void testSuccess() throws Exception {
     runSuccessfulUpdateTest("--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath);
@@ -52,7 +63,7 @@ public class UpdateReservedListCommandTest extends
 
   @Test
   public void testSuccess_lastUpdateTime_updatedCorrectly() throws Exception {
-    populateInitialReservedList(true);
+    populateInitialReservedListInDatastore(true);
     ReservedList original = ReservedList.get("xn--q9jyb4c_common-reserved").get();
     runCommandForced("--input=" + reservedTermsPath);
     ReservedList updated = ReservedList.get("xn--q9jyb4c_common-reserved").get();
@@ -71,7 +82,7 @@ public class UpdateReservedListCommandTest extends
 
   @Test
   public void testSuccess_shouldPublish_doesntOverrideFalseIfNotSpecified() throws Exception {
-    populateInitialReservedList(false);
+    populateInitialReservedListInDatastore(false);
     runCommandForced("--input=" + reservedTermsPath);
     assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent();
     ReservedList reservedList = ReservedList.get("xn--q9jyb4c_common-reserved").get();
@@ -79,7 +90,7 @@ public class UpdateReservedListCommandTest extends
   }
 
   private void runSuccessfulUpdateTest(String... args) throws Exception {
-    populateInitialReservedList(true);
+    populateInitialReservedListInDatastore(true);
     runCommandForced(args);
     assertThat(ReservedList.get("xn--q9jyb4c_common-reserved")).isPresent();
     ReservedList reservedList = ReservedList.get("xn--q9jyb4c_common-reserved").get();
@@ -100,4 +111,25 @@ public class UpdateReservedListCommandTest extends
                 runCommand("--force", "--name=xn--q9jyb4c_poobah", "--input=" + reservedTermsPath));
     assertThat(thrown).hasMessageThat().contains(errorMessage);
   }
+
+  @Test
+  public void testSaveToCloudSql_succeeds() throws Exception {
+    populateInitialReservedListInDatastore(true);
+    populateInitialReservedListInCloudSql(true);
+    runCommandForced(
+        "--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath, "--also_cloud_sql");
+    verifyXnq9jyb4cInDatastore();
+    verifyXnq9jyb4cInCloudSql();
+  }
+
+  @Test
+  public void testSaveToCloudSql_noExceptionThrownWhenSaveFail() throws Exception {
+    // Note that, during the dual-write phase, we want to make sure that no exception will be
+    // thrown if saving reserved list to Cloud SQL fails.
+    populateInitialReservedListInDatastore(true);
+    runCommandForced(
+        "--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath, "--also_cloud_sql");
+    verifyXnq9jyb4cInDatastore();
+    assertThat(ReservedListDao.checkExists("xn--q9jyb4c_common-reserved")).isFalse();
+  }
 }