Add Java code for storing and using IDN tables per-TLD (#1977)

This includes changes to make sure that we use the proper per-TLD IDN
tables as well as setting/updating/removing them via the Create/Update
TLD commands.

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -176,8 +176,7 @@ public class DomainFlowUtils {
       CharMatcher.inRange('a', 'z').or(CharMatcher.inRange('0', '9').or(CharMatcher.anyOf("-.")));
 
   /** Default validator used to determine if an IDN name can be provisioned on a TLD. */
-  private static final IdnLabelValidator IDN_LABEL_VALIDATOR =
-      IdnLabelValidator.createDefaultIdnLabelValidator();
+  private static final IdnLabelValidator IDN_LABEL_VALIDATOR = new IdnLabelValidator();
 
   /** The maximum number of DS records allowed on a domain. */
   private static final int MAX_DS_RECORDS_PER_DOMAIN = 8;

core/src/main/java/google/registry/model/tld/Registry.java

@@ -52,6 +52,7 @@ import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.ReservedList;
 import google.registry.persistence.VKey;
 import google.registry.persistence.converter.JodaMoneyType;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import google.registry.util.Idn;
 import java.util.List;
 import java.util.Map;
@@ -487,6 +488,9 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
    */
   List<VKey<AllocationToken>> defaultPromoTokens;
 
+  /** A set of allowed {@link IdnTableEnum}s for this TLD, or empty if we should use the default. */
+  Set<IdnTableEnum> idnTables;
+
   public String getTldStr() {
     return tldStr;
   }
@@ -694,6 +698,10 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
     return nullToEmptyImmutableCopy(defaultPromoTokens);
   }
 
+  public ImmutableSet<IdnTableEnum> getIdnTables() {
+    return nullToEmptyImmutableCopy(idnTables);
+  }
+
   @Override
   public Builder asBuilder() {
     return new Builder(clone(this));
@@ -992,6 +1000,11 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
       return this;
     }
 
+    public Builder setIdnTables(ImmutableSet<IdnTableEnum> idnTables) {
+      getInstance().idnTables = idnTables;
+      return this;
+    }
+
     @Override
     public Registry build() {
       final Registry instance = getInstance();

core/src/main/java/google/registry/persistence/converter/IdnTableEnumSetConverter.java

@@ -0,0 +1,34 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import google.registry.tldconfig.idn.IdnTableEnum;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA {@link AttributeConverter} for storing/retrieving {@link IdnTableEnum}s. */
+@Converter(autoApply = true)
+public class IdnTableEnumSetConverter extends StringSetConverterBase<IdnTableEnum> {
+
+  @Override
+  String toString(IdnTableEnum element) {
+    return element.name();
+  }
+
+  @Override
+  IdnTableEnum fromString(String value) {
+    return IdnTableEnum.valueOf(value);
+  }
+}

core/src/main/java/google/registry/tldconfig/idn/IdnLabelValidator.java

@@ -17,8 +17,8 @@ package google.registry.tldconfig.idn;
 import static google.registry.tldconfig.idn.IdnTableEnum.EXTENDED_LATIN;
 import static google.registry.tldconfig.idn.IdnTableEnum.JA;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.tld.Registry;
 import google.registry.util.Idn;
 import java.util.Optional;
 
@@ -26,23 +26,8 @@ import java.util.Optional;
 public final class IdnLabelValidator {
 
   /** Most TLDs will use this generic list of IDN tables. */
-  private static final ImmutableList<IdnTableEnum> DEFAULT_IDN_TABLES =
-      ImmutableList.of(EXTENDED_LATIN, JA);
-
-  private static final ImmutableMap<String, ImmutableList<IdnTableEnum>>
-      DEFAULT_IDN_TABLE_LISTS_PER_TLD =
-          ImmutableMap.of("xn--q9jyb4c", ImmutableList.of(EXTENDED_LATIN, JA));
-
-  /** Some TLDs have their own IDN tables, configured here. */
-  private ImmutableMap<String, ImmutableList<IdnTableEnum>> idnTableListsPerTld;
-
-  IdnLabelValidator(ImmutableMap<String, ImmutableList<IdnTableEnum>> indTableListsPerTld) {
-    this.idnTableListsPerTld = indTableListsPerTld;
-  }
-
-  public static IdnLabelValidator createDefaultIdnLabelValidator() {
-    return new IdnLabelValidator(DEFAULT_IDN_TABLE_LISTS_PER_TLD);
-  }
+  private static final ImmutableSet<IdnTableEnum> DEFAULT_IDN_TABLES =
+      ImmutableSet.of(EXTENDED_LATIN, JA);
 
   /**
    * Returns name of first matching {@link IdnTable} if domain label is valid for the given TLD.
@@ -50,10 +35,13 @@ public final class IdnLabelValidator {
    * <p>A label is valid if it is considered valid by at least one configured IDN table for that
    * TLD. If no match is found, an absent value is returned.
    */
-  public Optional<String> findValidIdnTableForTld(String label, String tld) {
+  public Optional<String> findValidIdnTableForTld(String label, String tldStr) {
     String unicodeString = Idn.toUnicode(label);
-    for (IdnTableEnum idnTable :
-        Optional.ofNullable(idnTableListsPerTld.get(tld)).orElse(DEFAULT_IDN_TABLES)) {
+    Registry tld = Registry.get(tldStr); // uses the cache
+    ImmutableSet<IdnTableEnum> idnTablesForTld = tld.getIdnTables();
+    ImmutableSet<IdnTableEnum> idnTables =
+        idnTablesForTld.isEmpty() ? DEFAULT_IDN_TABLES : idnTablesForTld;
+    for (IdnTableEnum idnTable : idnTables) {
       if (idnTable.getTable().isValidLabel(unicodeString)) {
         return Optional.of(idnTable.getTable().getName());
       }

core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java

@@ -15,6 +15,7 @@
 package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.tools.UpdateOrDeleteAllocationTokensCommand.getTokenKeys;
 import static google.registry.util.CollectionUtils.findDuplicates;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
@@ -34,9 +35,11 @@ import google.registry.model.tld.Registry.TldState;
 import google.registry.model.tld.Registry.TldType;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.PremiumListDao;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import google.registry.tools.params.OptionalStringParameter;
 import google.registry.tools.params.TransitionListParameter.BillingCostTransitions;
 import google.registry.tools.params.TransitionListParameter.TldStateTransitions;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -244,6 +247,15 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
               + " present default tokens.")
   List<String> defaultTokens;
 
+  @Nullable
+  @Parameter(
+      names = "--idn_tables",
+      description =
+          "A comma-separated list of the IDN tables to use for this TLD. Specify an empty list to"
+              + " remove any previously-set tables and to use the default. All elements must be"
+              + " IdnTableEnum values")
+  List<String> idnTables;
+
   /** Returns the existing registry (for update) or null (for creates). */
   @Nullable
   abstract Registry getOldRegistry(String tld);
@@ -392,6 +404,23 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
           builder.setDefaultPromoTokens(getTokenKeys(defaultTokens, null));
         }
       }
+      if (idnTables != null) {
+        if (idnTables.equals(ImmutableList.of(""))) {
+          builder.setIdnTables(ImmutableSet.of());
+        } else {
+          ImmutableSet<String> upperCaseIdnTables =
+              idnTables.stream().map(String::toUpperCase).collect(toImmutableSet());
+          ImmutableSet<String> validIdnStringValues =
+              Arrays.stream(IdnTableEnum.values()).map(Enum::name).collect(toImmutableSet());
+          checkArgument(
+              validIdnStringValues.containsAll(upperCaseIdnTables),
+              "IDN tables %s contained invalid value(s). Possible values: %s",
+              upperCaseIdnTables,
+              validIdnStringValues);
+          builder.setIdnTables(
+              upperCaseIdnTables.stream().map(IdnTableEnum::valueOf).collect(toImmutableSet()));
+        }
+      }
       // Update the Registry object.
       setCommandSpecificProperties(builder);
       stageEntityChange(oldRegistry, builder.build());

core/src/main/resources/META-INF/persistence.xml

@@ -91,6 +91,7 @@
     <class>google.registry.persistence.converter.DatabaseMigrationScheduleTransitionConverter</class>
     <class>google.registry.persistence.converter.DateTimeConverter</class>
     <class>google.registry.persistence.converter.DurationConverter</class>
+    <class>google.registry.persistence.converter.IdnTableEnumSetConverter</class>
     <class>google.registry.persistence.converter.InetAddressSetConverter</class>
     <class>google.registry.persistence.converter.LocalDateConverter</class>
     <class>google.registry.persistence.converter.PostalInfoChoiceListConverter</class>

core/src/test/java/google/registry/tldconfig/idn/IdnLabelValidatorTest.java

@@ -15,17 +15,26 @@
 package google.registry.tldconfig.idn;
 
 import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.persistResource;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link IdnLabelValidator}. */
 class IdnLabelValidatorTest {
 
-  private IdnLabelValidator idnLabelValidator = IdnLabelValidator.createDefaultIdnLabelValidator();
+  @RegisterExtension
+  final JpaIntegrationTestExtension jpa =
+      new JpaTestExtensions.Builder().buildIntegrationTestExtension();
 
-  private void doJapaneseLanguageTests(String tld) {
+  private IdnLabelValidator idnLabelValidator = new IdnLabelValidator();
+
+  private void doJapaneseAndLatinLanguageTests(String tld) {
+    createTld(tld);
     assertThat(idnLabelValidator.findValidIdnTableForTld("foo", tld)).isPresent();
     assertThat(idnLabelValidator.findValidIdnTableForTld("12379foar", tld)).isPresent();
     assertThat(idnLabelValidator.findValidIdnTableForTld("みんな", tld)).isPresent();
@@ -84,26 +93,29 @@ class IdnLabelValidatorTest {
 
   @Test
   void testMinna() {
-    doJapaneseLanguageTests("xn--q9jyb4c");
+    doJapaneseAndLatinLanguageTests("xn--q9jyb4c");
   }
 
   @Test
   void testFoo() {
-    doJapaneseLanguageTests("foo");
+    doJapaneseAndLatinLanguageTests("foo");
   }
 
   @Test
   void testSoy() {
-    doJapaneseLanguageTests("soy");
+    doJapaneseAndLatinLanguageTests("soy");
   }
 
   @Test
-  void testOverridenTables() {
-    // Set .tld to have only the extended latin table and not japanese.
-    idnLabelValidator =
-        new IdnLabelValidator(
-            ImmutableMap.of("tld", ImmutableList.of(IdnTableEnum.EXTENDED_LATIN)));
+  void testPerTldConfig() {
+    persistResource(
+        createTld("tld")
+            .asBuilder()
+            .setIdnTables(ImmutableSet.of(IdnTableEnum.EXTENDED_LATIN))
+            .build());
     assertThat(idnLabelValidator.findValidIdnTableForTld("foo", "tld")).isPresent();
+    assertThat(idnLabelValidator.findValidIdnTableForTld("abcdefghæ", "tld")).isPresent();
+    // Extended Latin shouldn't include Japanese characters
     assertThat(idnLabelValidator.findValidIdnTableForTld("みんな", "tld")).isEmpty();
   }
 }

core/src/test/java/google/registry/tools/CreateTldCommandTest.java

@@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Range;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.tld.Registry;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import java.math.BigDecimal;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -544,6 +545,35 @@ class CreateTldCommandTest extends CommandTestCase<CreateTldCommand> {
     assertThat(Registry.get("xn--q9jyb4c").getDriveFolderId()).isNull();
   }
 
+  @Test
+  void testSuccess_setsIdnTables() throws Exception {
+    runCommandForced(
+        "--idn_tables=extended_latin,ja",
+        "--roid_suffix=ASDF",
+        "--dns_writers=VoidDnsWriter",
+        "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables())
+        .containsExactly(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA);
+  }
+
+  @Test
+  void testFailure_invalidIdnTable() throws Exception {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--idn_tables=extended_latin,bad_value",
+                    "--roid_suffix=ASDF",
+                    "--dns_writers=VoidDnsWriter",
+                    "xn--q9jyb4c"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "IDN tables [EXTENDED_LATIN, BAD_VALUE] contained invalid value(s). Possible values:"
+                + " [EXTENDED_LATIN, UNCONFUSABLE_LATIN, JA]");
+  }
+
   @Test
   void testFailure_setPremiumListThatDoesntExist() {
     IllegalArgumentException thrown =

core/src/test/java/google/registry/tools/UpdateTldCommandTest.java

@@ -39,6 +39,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.tld.Registry;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -1053,6 +1054,38 @@ class UpdateTldCommandTest extends CommandTestCase<UpdateTldCommand> {
     assertThat(Registry.get("xn--q9jyb4c").getDriveFolderId()).isNull();
   }
 
+  @Test
+  void testSuccess_setsIdnTables() throws Exception {
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables()).isEmpty();
+    runCommandForced("--idn_tables=extended_latin,ja", "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables())
+        .containsExactly(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA);
+  }
+
+  @Test
+  void testSuccess_removesIndTables() throws Exception {
+    persistResource(
+        Registry.get("xn--q9jyb4c")
+            .asBuilder()
+            .setIdnTables(ImmutableSet.of(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA))
+            .build());
+    runCommandForced("--idn_tables=", "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables()).isEmpty();
+  }
+
+  @Test
+  void testFailure_invalidIdnTable() throws Exception {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> runCommandForced("--idn_tables=extended_latin,bad_value", "xn--q9jyb4c"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "IDN tables [EXTENDED_LATIN, BAD_VALUE] contained invalid value(s). Possible values:"
+                + " [EXTENDED_LATIN, UNCONFUSABLE_LATIN, JA]");
+  }
+
   @Test
   void testFailure_setPremiumListThatDoesntExist() {
     IllegalArgumentException thrown =

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -724,6 +724,7 @@
         drive_folder_id text,
         eap_fee_schedule hstore not null,
         escrow_enabled boolean not null,
+        idn_tables text[],
         invoicing_enabled boolean not null,
         lordn_username text,
         num_dns_publish_locks int4 not null,