Canonicalize domain/host names in async DS->SQL replay (#1350)

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -24,6 +24,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import java.util.Set;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -164,6 +165,11 @@ public class DomainBase extends DomainContent
     return cloneDomainProjectedAtTime(this, now);
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -33,6 +33,7 @@ import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.HashSet;
 import java.util.Optional;
@@ -303,6 +304,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   public void beforeSqlSaveOnReplay() {
     if (domainContent == null) {
       domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
+      domainContent.fullyQualifiedDomainName =
+          DomainNameUtils.canonicalizeDomainName(domainContent.fullyQualifiedDomainName);
       fillAuxiliaryFieldsFromDomain(this);
     }
   }

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -26,6 +26,7 @@ import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -141,6 +142,8 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial
   public void beforeSqlSaveOnReplay() {
     if (hostBase == null) {
       hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
+      hostBase.fullyQualifiedHostName =
+          DomainNameUtils.canonicalizeDomainName(hostBase.fullyQualifiedHostName);
     }
   }
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -22,6 +22,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 
@@ -51,6 +52,11 @@ public class HostResource extends HostBase
     return VKey.create(HostResource.class, getRepoId(), Key.create(this));
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedHostName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedHostName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -921,6 +921,19 @@ public class DomainBaseTest extends EntityTestCase {
         .containsExactly(EppResourceIndex.create(Key.create(domain)));
   }
 
+  @Test
+  void testBeforeSqlSaveOnReplay_canonicalName() {
+    domain.fullyQualifiedDomainName = "EXAMPLE.COM";
+    assertThat(domain.getDomainName()).isEqualTo("EXAMPLE.COM");
+    domain.beforeSqlSaveOnReplay();
+    assertThat(domain.getDomainName()).isEqualTo("example.com");
+
+    domain.fullyQualifiedDomainName = "kittyçat.com";
+    assertThat(domain.getDomainName()).isEqualTo("kittyçat.com");
+    domain.beforeSqlSaveOnReplay();
+    assertThat(domain.getDomainName()).isEqualTo("xn--kittyat-yxa.com");
+  }
+
   static class BillEventInfo extends ImmutableObject {
     VKey<BillingEvent.Recurring> billingEventRecurring;
     Long billingEventRecurringHistoryId;

core/src/test/java/google/registry/model/history/DomainHistoryTest.java

@@ -56,6 +56,7 @@ import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.TestOfyOnly;
 import google.registry.testing.TestSqlOnly;
 import google.registry.util.SerializeUtils;
+import java.lang.reflect.Field;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
@@ -136,8 +137,7 @@ public class DomainHistoryTest extends EntityTestCase {
     DomainHistory domainHistory = createDomainHistory(domain);
     tm().transact(() -> tm().insert(domainHistory));
 
-    // retrieving a HistoryEntry or a DomainHistory with the same key should return the same
+    // retrieving a HistoryEntry or a DomainHistory with the same key should return the same object
-    // object
     // note: due to the @EntitySubclass annotation. all Keys for DomainHistory objects will have
     // type HistoryEntry
     VKey<DomainHistory> domainHistoryVKey = domainHistory.createVKey();
@@ -232,6 +232,66 @@ public class DomainHistoryTest extends EntityTestCase {
                         jpaTm().loadByEntity(historyWithoutResource).getDomainContent().get()));
   }
 
+  @TestSqlOnly
+  void testBeforeSqlSave_canonicalNameUncapitalized() throws Exception {
+    Field domainNameField = DomainContent.class.getDeclaredField("fullyQualifiedDomainName");
+    // reflection hacks to get around visibility issues
+    domainNameField.setAccessible(true);
+    DomainBase domain = createDomainWithContactsAndHosts();
+    domainNameField.set(domain, "EXAMPLE.TLD");
+
+    DomainHistory historyWithoutResource =
+        new DomainHistory.Builder()
+            .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .setModificationTime(fakeClock.nowUtc())
+            .setRegistrarId("TheRegistrar")
+            .setTrid(Trid.create("ABC-123", "server-trid"))
+            .setBySuperuser(false)
+            .setReason("reason")
+            .setRequestedByRegistrar(true)
+            .setDomainRepoId(domain.getRepoId())
+            .setOtherRegistrarId("otherClient")
+            .setPeriod(Period.create(1, Period.Unit.YEARS))
+            .build();
+
+    DatabaseHelper.putInDb(domain, historyWithoutResource);
+    jpaTm().transact(historyWithoutResource::beforeSqlSaveOnReplay);
+
+    assertThat(historyWithoutResource.getDomainContent().get().getDomainName())
+        .isEqualTo("example.tld");
+  }
+
+  @TestSqlOnly
+  void testBeforeSqlSave_canonicalNameUtf8() throws Exception {
+    Field domainNameField = DomainContent.class.getDeclaredField("fullyQualifiedDomainName");
+    // reflection hacks to get around visibility issues
+    domainNameField.setAccessible(true);
+    DomainBase domain = createDomainWithContactsAndHosts();
+    domainNameField.set(domain, "kittyçat.tld");
+
+    DomainHistory historyWithoutResource =
+        new DomainHistory.Builder()
+            .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .setModificationTime(fakeClock.nowUtc())
+            .setRegistrarId("TheRegistrar")
+            .setTrid(Trid.create("ABC-123", "server-trid"))
+            .setBySuperuser(false)
+            .setReason("reason")
+            .setRequestedByRegistrar(true)
+            .setDomainRepoId(domain.getRepoId())
+            .setOtherRegistrarId("otherClient")
+            .setPeriod(Period.create(1, Period.Unit.YEARS))
+            .build();
+
+    DatabaseHelper.putInDb(domain, historyWithoutResource);
+    jpaTm().transact(historyWithoutResource::beforeSqlSaveOnReplay);
+
+    assertThat(historyWithoutResource.getDomainContent().get().getDomainName())
+        .isEqualTo("xn--kittyat-yxa.tld");
+  }
+
   static DomainBase createDomainWithContactsAndHosts() {
     createTld("tld");
     HostResource host = newHostResourceWithRoid("ns1.example.com", "host1");

core/src/test/java/google/registry/model/history/HostHistoryTest.java

@@ -33,10 +33,12 @@ import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.TestOfyOnly;
 import google.registry.testing.TestSqlOnly;
 import google.registry.util.SerializeUtils;
+import java.lang.reflect.Field;
 
 /** Tests for {@link HostHistory}. */
 @DualDatabaseTest
@@ -146,6 +148,58 @@ public class HostHistoryTest extends EntityTestCase {
                     .hasFieldsEqualTo(jpaTm().loadByEntity(hostHistory).getHostBase().get()));
   }
 
+  @TestSqlOnly
+  void testBeforeSqlSave_canonicalNameUncapitalized() throws Exception {
+    Field hostNameField = HostBase.class.getDeclaredField("fullyQualifiedHostName");
+    // reflection hacks to get around visibility issues
+    hostNameField.setAccessible(true);
+    HostResource hostResource = newHostResource("ns1.example.tld");
+    hostNameField.set(hostResource, "NS1.EXAMPLE.TLD");
+    HostHistory hostHistory =
+        new HostHistory.Builder()
+            .setType(HistoryEntry.Type.HOST_CREATE)
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .setModificationTime(fakeClock.nowUtc())
+            .setRegistrarId("TheRegistrar")
+            .setTrid(Trid.create("ABC-123", "server-trid"))
+            .setBySuperuser(false)
+            .setReason("reason")
+            .setRequestedByRegistrar(true)
+            .setHostRepoId(hostResource.getRepoId())
+            .build();
+
+    DatabaseHelper.putInDb(hostResource, hostHistory);
+    jpaTm().transact(hostHistory::beforeSqlSaveOnReplay);
+
+    assertThat(hostHistory.getHostBase().get().getHostName()).isEqualTo("ns1.example.tld");
+  }
+
+  @TestSqlOnly
+  void testBeforeSqlSave_canonicalNameUtf8() throws Exception {
+    Field hostNameField = HostBase.class.getDeclaredField("fullyQualifiedHostName");
+    // reflection hacks to get around visibility issues
+    hostNameField.setAccessible(true);
+    HostResource hostResource = newHostResource("ns1.example.tld");
+    hostNameField.set(hostResource, "ns1.kittyçat.tld");
+    HostHistory hostHistory =
+        new HostHistory.Builder()
+            .setType(HistoryEntry.Type.HOST_CREATE)
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .setModificationTime(fakeClock.nowUtc())
+            .setRegistrarId("TheRegistrar")
+            .setTrid(Trid.create("ABC-123", "server-trid"))
+            .setBySuperuser(false)
+            .setReason("reason")
+            .setRequestedByRegistrar(true)
+            .setHostRepoId(hostResource.getRepoId())
+            .build();
+
+    DatabaseHelper.putInDb(hostResource, hostHistory);
+    jpaTm().transact(hostHistory::beforeSqlSaveOnReplay);
+
+    assertThat(hostHistory.getHostBase().get().getHostName()).isEqualTo("ns1.xn--kittyat-yxa.tld");
+  }
+
   private void assertHostHistoriesEqual(HostHistory one, HostHistory two) {
     assertAboutImmutableObjects().that(one).isEqualExceptFields(two, "hostBase");
     assertAboutImmutableObjects()

core/src/test/java/google/registry/model/host/HostResourceTest.java

@@ -319,4 +319,17 @@ class HostResourceTest extends EntityTestCase {
     assertThat(ofyTm().loadAllOf(EppResourceIndex.class))
         .containsExactly(EppResourceIndex.create(Key.create(host)));
   }
+
+  @TestOfyOnly
+  void testBeforeSqlSaveOnReplay_canonicalName() {
+    host.fullyQualifiedHostName = "NS1.EXAMPLE.COM";
+    assertThat(host.getHostName()).isEqualTo("NS1.EXAMPLE.COM");
+    host.beforeSqlSaveOnReplay();
+    assertThat(host.getHostName()).isEqualTo("ns1.example.com");
+
+    host.fullyQualifiedHostName = "ns1.kittyçat.com";
+    assertThat(host.getHostName()).isEqualTo("ns1.kittyçat.com");
+    host.beforeSqlSaveOnReplay();
+    assertThat(host.getHostName()).isEqualTo("ns1.xn--kittyat-yxa.com");
+  }
 }