Begin migration away from ReferenceUnions

It is important to get at least this one commit in before the public Nomulus
release so that none of our public users will have to go through this data
migration (although we will have to).

The migration strategy is as follows:

1. Dual-write to non-ReferenceUnion fields in addition to the current
ReferenceUnion fields in use, and add new indexes (this commit). Deploy.
2. Run the ResaveAllEppResourcesAction backfill [].
3. Switch all code over to using the new fields. Dual-write is still in effect,
except it is now copying over the values of the new fields to the old
fields. Switch over all BigQuery reporting scripts to use the new
fields. Deploy.
4. Remove all of the old code and indexes. Deploy.
5. (Optional, at our leisure) Re-run the ResaveAllEppResourcesAction backfill
[] to delete the old obsolete fields.

Note that this migration strategy is rollback-safe at every step -- new data is
not read until it has already been written out in the previous step, and old
data is not removed immediately following a step in which it was still being
read, so the previous step is safe to roll back to.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=136196988

java/google/registry/env/common/default/WEB-INF/datastore-indexes.xml

@@ -10,7 +10,7 @@
         <property name="currentSponsorClientId" direction="asc"/>
         <property name="deletionTime" direction="asc"/>
     </datastore-index>
-    <!-- For finding domain resources by tld. -->
+    <!-- For finding domain resources by TLD. -->
     <datastore-index kind="DomainBase" ancestor="false" source="manual">
         <property name="^i" direction="asc"/>
         <property name="tld" direction="asc"/>
@@ -26,28 +26,44 @@
         <property name="currentSponsorClientId" direction="asc"/>
         <property name="deletionTime" direction="asc"/>
     </datastore-index>
-    <!-- For finding account balance of Registrar and viewing billing history. -->
+    <!-- For finding account balance of registrar and viewing billing history. -->
     <datastore-index kind="RegistrarBillingEntry" ancestor="true" source="manual">
         <property name="currency" direction="asc"/>
         <property name="created" direction="desc"/>
     </datastore-index>
     <!-- For determining the active domains linked to a given contact. -->
+    <datastore-index kind="DomainBase" ancestor="false" source="manual">
+        <property name="allContacts.contact" direction="asc"/>
+        <property name="deletionTime" direction="asc"/>
+    </datastore-index>
+    <!-- TODO(b/28713909): Remove this index along with ReferenceUnions. -->
     <datastore-index kind="DomainBase" ancestor="false" source="manual">
         <property name="allContacts.contactId.linked" direction="asc"/>
         <property name="deletionTime" direction="asc"/>
     </datastore-index>
     <!-- For determining the active domains linked to a given host. -->
+    <datastore-index kind="DomainBase" ancestor="false" source="manual">
+        <property name="nsHosts" direction="asc"/>
+        <property name="deletionTime" direction="asc"/>
+    </datastore-index>
+    <!-- TODO(b/28713909): Remove this index along with ReferenceUnions. -->
     <datastore-index kind="DomainBase" ancestor="false" source="manual">
         <property name="nameservers.linked" direction="asc"/>
         <property name="deletionTime" direction="asc"/>
     </datastore-index>
-    <!-- For updating domains and applications after a host rename. -->
+    <!-- For RDAP searches by linked nameserver. -->
+    <datastore-index kind="DomainBase" ancestor="false" source="manual">
+        <property name="^i" direction="asc"/>
+        <property name="nsHosts" direction="asc"/>
+        <property name="deletionTime" direction="asc"/>
+    </datastore-index>
+    <!-- TODO(b/28713909): Remove this index along with ReferenceUnions. -->
     <datastore-index kind="DomainBase" ancestor="false" source="manual">
         <property name="^i" direction="asc"/>
         <property name="nameservers.linked" direction="asc"/>
         <property name="deletionTime" direction="asc"/>
     </datastore-index>
-    <!-- For Whois ip lookup -->
+    <!-- For WHOIS IP address lookup -->
     <datastore-index kind="HostResource" ancestor="false" source="manual">
         <property name="inetAddresses" direction="asc"/>
         <property name="deletionTime" direction="asc"/>

java/google/registry/model/domain/DesignatedContact.java

@@ -52,6 +52,7 @@ public class DesignatedContact extends ImmutableObject {
     DesignatedContact instance = new DesignatedContact();
     instance.type = type;
     instance.contactId = ReferenceUnion.create(contact);
+    instance.contact = contact;
     return instance;
   }
 
@@ -60,9 +61,12 @@ public class DesignatedContact extends ImmutableObject {
 
   @Index
   @XmlValue
-  //TODO(b/28713909): Make this a Key<ContactResource>.
+  //TODO(b/28713909): Remove contactId and replace with contact.
   ReferenceUnion<ContactResource> contactId;
 
+  @Index
+  Key<ContactResource> contact;
+
   public Type getType() {
     return type;
   }

java/google/registry/model/domain/DomainBase.java

@@ -40,6 +40,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnSave;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactResource;
@@ -78,9 +79,13 @@ public abstract class DomainBase extends EppResource {
 
   /** References to hosts that are the nameservers for the domain. */
   @XmlTransient
-  //TODO(b/28713909): Make this a Set<Key<HostResource>>.
+  //TODO(b/28713909): Delete this once migration away from ReferenceUnions is complete.
   Set<ReferenceUnion<HostResource>> nameservers;
 
+  @Index
+  @XmlTransient
+  Set<Key<HostResource>> nsHosts;
+
   /**
    * The union of the contacts visible via {@link #getContacts} and {@link #getRegistrant}.
    *
@@ -234,6 +239,18 @@ public abstract class DomainBase extends EppResource {
     return tld;
   }
 
+  @OnSave
+  void dualSaveReferenceUnions() {
+    for (DesignatedContact contact : nullToEmptyImmutableCopy(allContacts)) {
+      contact.contact = contact.contactId.getLinked();
+    }
+    ImmutableSet.Builder<Key<HostResource>> hostKeys = new ImmutableSet.Builder<>();
+    for (ReferenceUnion<HostResource> refUnion : nullToEmptyImmutableCopy(nameservers)) {
+      hostKeys.add(refUnion.getLinked());
+    }
+    nsHosts = hostKeys.build();
+  }
+
   /** Predicate to determine if a given {@link DesignatedContact} is the registrant. */
   private static final Predicate<DesignatedContact> IS_REGISTRANT =
       new Predicate<DesignatedContact>() {

java/google/registry/model/domain/ReferenceUnion.java

@@ -26,7 +26,9 @@ import google.registry.model.ImmutableObject;
  *
  * @param <T> the type being referenced
  */
+// TODO(b/28713909): Delete ReferenceUnion entirely.
 @Embed
+@Deprecated
 public class ReferenceUnion<T extends EppResource> extends ImmutableObject {
 
   @Index

javatests/google/registry/model/domain/DomainApplicationTest.java

@@ -120,8 +120,10 @@ public class DomainApplicationTest extends EntityTestCase {
     verifyIndexing(
         domainApplication,
         "allContacts.contactId.linked",
+        "allContacts.contact",
         "fullyQualifiedDomainName",
         "nameservers.linked",
+        "nsHosts",
         "deletionTime",
         "currentSponsorClientId",
         "tld");

javatests/google/registry/model/domain/DomainResourceTest.java

@@ -23,6 +23,8 @@ import static google.registry.testing.DatastoreHelper.cloneAndSetAutoTimestamps;
 import static google.registry.testing.DatastoreHelper.createTld;
 import static google.registry.testing.DatastoreHelper.newDomainResource;
 import static google.registry.testing.DatastoreHelper.newHostResource;
+import static google.registry.testing.DatastoreHelper.persistActiveContact;
+import static google.registry.testing.DatastoreHelper.persistActiveHost;
 import static google.registry.testing.DatastoreHelper.persistResource;
 import static google.registry.testing.DomainResourceSubject.assertAboutDomains;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -174,8 +176,10 @@ public class DomainResourceTest extends EntityTestCase {
     verifyIndexing(
         domain,
         "allContacts.contactId.linked",
+        "allContacts.contact",
         "fullyQualifiedDomainName",
         "nameservers.linked",
+        "nsHosts",
         "currentSponsorClientId",
         "deletionTime",
         "tld");
@@ -452,4 +456,27 @@ public class DomainResourceTest extends EntityTestCase {
   public void testToHydratedString_notCircular() {
     domain.toHydratedString();  // If there are circular references, this will overflow the stack.
   }
+
+  // TODO(b/28713909): Remove these tests once ReferenceUnion migration is complete.
+  @Test
+  public void testDualSavingOfDesignatedContact() {
+    ContactResource contact = persistActiveContact("time1006");
+    DesignatedContact designatedContact = new DesignatedContact();
+    designatedContact.contactId = ReferenceUnion.create(Key.create(contact));
+    designatedContact.type = Type.ADMIN;
+    DomainResource domainWithContact =
+        domain.asBuilder().setContacts(ImmutableSet.of(designatedContact)).build();
+    assertThat(getOnlyElement(domainWithContact.getContacts()).contact).isNull();
+    DomainResource reloadedDomain = persistResource(domainWithContact);
+    assertThat(getOnlyElement(reloadedDomain.getContacts()).contact).isEqualTo(Key.create(contact));
+  }
+
+  @Test
+  public void testDualSavingOfNameservers() {
+    HostResource host = persistActiveHost("zzz.xxx.yyy");
+    DomainResource domain = newDomainResource("python-django-unchained.com", host);
+    assertThat(domain.nsHosts).isNull();
+    DomainResource djangoReloaded = persistResource(domain);
+    assertThat(djangoReloaded.nsHosts).containsExactly(Key.create(host));
+  }
 }

javatests/google/registry/model/schema.txt

@@ -187,6 +187,7 @@ enum google.registry.model.contact.PostalInfo$Type {
   LOCALIZED;
 }
 class google.registry.model.domain.DesignatedContact {
+  com.googlecode.objectify.Key<google.registry.model.contact.ContactResource> contact;
   google.registry.model.domain.DesignatedContact$Type type;
   google.registry.model.domain.ReferenceUnion<google.registry.model.contact.ContactResource> contactId;
 }
@@ -214,6 +215,7 @@ class google.registry.model.domain.DomainApplication {
   java.lang.String lastEppUpdateClientId;
   java.lang.String tld;
   java.util.List<google.registry.model.smd.EncodedSignedMark> encodedSignedMarks;
+  java.util.Set<com.googlecode.objectify.Key<google.registry.model.host.HostResource>> nsHosts;
   java.util.Set<google.registry.model.domain.DesignatedContact> allContacts;
   java.util.Set<google.registry.model.domain.ReferenceUnion<google.registry.model.host.HostResource>> nameservers;
   java.util.Set<google.registry.model.domain.secdns.DelegationSignerData> dsData;
@@ -240,6 +242,7 @@ class google.registry.model.domain.DomainBase {
   java.lang.String idnTableName;
   java.lang.String lastEppUpdateClientId;
   java.lang.String tld;
+  java.util.Set<com.googlecode.objectify.Key<google.registry.model.host.HostResource>> nsHosts;
   java.util.Set<google.registry.model.domain.DesignatedContact> allContacts;
   java.util.Set<google.registry.model.domain.ReferenceUnion<google.registry.model.host.HostResource>> nameservers;
   java.util.Set<google.registry.model.domain.secdns.DelegationSignerData> dsData;
@@ -267,6 +270,7 @@ class google.registry.model.domain.DomainResource {
   java.lang.String lastEppUpdateClientId;
   java.lang.String smdId;
   java.lang.String tld;
+  java.util.Set<com.googlecode.objectify.Key<google.registry.model.host.HostResource>> nsHosts;
   java.util.Set<google.registry.model.domain.DesignatedContact> allContacts;
   java.util.Set<google.registry.model.domain.GracePeriod> gracePeriods;
   java.util.Set<google.registry.model.domain.ReferenceUnion<google.registry.model.host.HostResource>> nameservers;