diff --git a/core/src/main/java/google/registry/beam/initsql/Transforms.java b/core/src/main/java/google/registry/beam/initsql/Transforms.java index 60a26b499..df082cdbf 100644 --- a/core/src/main/java/google/registry/beam/initsql/Transforms.java +++ b/core/src/main/java/google/registry/beam/initsql/Transforms.java @@ -32,9 +32,12 @@ import com.google.appengine.api.datastore.EntityTranslator; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import google.registry.backup.CommitLogImports; import google.registry.backup.VersionedEntity; +import google.registry.model.billing.BillingEvent.Flag; +import google.registry.model.billing.BillingEvent.Reason; import google.registry.model.domain.DomainBase; import google.registry.model.reporting.HistoryEntry; import google.registry.schema.replay.DatastoreAndSqlEntity; @@ -256,7 +259,58 @@ public final class Transforms { .iterator())); } + // Production data repair configs go below. See b/185954992. + + // Prober domains in bad state, without associated contacts, hosts, billings, and history. + // They can be safely ignored. + private static final ImmutableSet IGNORED_DOMAINS = + ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT"); + + // Prober hosts referencing phantom registrars. They and their associated history entries can be + // safely ignored. + private static final ImmutableSet IGNORED_HOSTS = + ImmutableSet.of( + "4E21_WJ0TEST-GOOGLE", + "4E21_WJ1TEST-GOOGLE", + "4E21_WJ2TEST-GOOGLE", + "4E21_WJ3TEST-GOOGLE"); + + // Prober contacts referencing phantom registrars. They and their associated history entries can + // be safely ignored. + private static final ImmutableSet IGNORED_CONTACTS = + ImmutableSet.of( + "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE"); + private static boolean isMigratable(Entity entity) { + // Checks specific to production data. See b/185954992 for details. + // The names of these bad entities in production do not conflict with other environments. For + // simplicities sake we apply them regardless of the source of the data. + if (entity.getKind().equals("DomainBase") + && IGNORED_DOMAINS.contains(entity.getKey().getName())) { + return false; + } + if (entity.getKind().equals("ContactResource")) { + String roid = entity.getKey().getName(); + return !IGNORED_CONTACTS.contains(roid); + } + if (entity.getKind().equals("HostResource")) { + String roid = entity.getKey().getName(); + return !IGNORED_HOSTS.contains(roid); + } + if (entity.getKind().equals("HistoryEntry")) { + // Remove production bad data: History of the contacts to be ignored: + com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent(); + if (parentKey.getKind().equals("ContactResource")) { + String contactRoid = parentKey.getName(); + return !IGNORED_CONTACTS.contains(contactRoid); + } + if (parentKey.getKind().equals("HostResource")) { + String hostRoid = parentKey.getName(); + return !IGNORED_HOSTS.contains(hostRoid); + } + } + // End of production-specific checks. + if (entity.getKind().equals("HistoryEntry")) { // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated. // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot @@ -266,6 +320,18 @@ public final class Transforms { return true; } + private static Entity repairBadData(Entity entity) { + if (entity.getKind().equals("Cancellation") + && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) { + // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the + // AUTO_RENEW flag. Note: all affected entities have empty flags so we can simply assign + // instead of append. See b/185954992. + entity.setUnindexedProperty("reason", Reason.RENEW.name()); + entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name())); + } + return entity; + } + private static SqlEntity toSqlEntity(Object ofyEntity) { if (ofyEntity instanceof HistoryEntry) { HistoryEntry ofyHistory = (HistoryEntry) ofyEntity; @@ -286,6 +352,7 @@ public final class Transforms { return dsEntity .getEntity() .filter(Transforms::isMigratable) + .map(Transforms::repairBadData) .map(e -> ofy().toPojo(e)) .map(Transforms::toSqlEntity) .orElse(null);