diff --git a/core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java b/core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java index d3a33828d..6b55ef6e6 100644 --- a/core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java +++ b/core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java @@ -104,6 +104,7 @@ final class ValidateSqlUtils { private final HashMap missingCounters = new HashMap<>(); private final HashMap unequalCounters = new HashMap<>(); private final HashMap badEntityCounters = new HashMap<>(); + private final HashMap duplicateEntityCounters = new HashMap<>(); private volatile boolean logPrinted = false; @@ -120,6 +121,8 @@ final class ValidateSqlUtils { counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey)); unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey)); badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey)); + duplicateEntityCounters.put( + counterKey, Metrics.counter("CompareDB", "Duplicate Entities:" + counterKey)); } /** @@ -158,12 +161,18 @@ final class ValidateSqlUtils { ImmutableList entities = ImmutableList.copyOf(kv.getValue()); verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey()); - verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey()); String counterKey = getCounterKey(entities.get(0).getClass()); ensureCounterExists(counterKey); totalCounters.get(counterKey).inc(); + if (entities.size() > 2) { + // Duplicates may happen with Cursors if imported across projects. Its key in Datastore, the + // id field, encodes the project name and is not fixed by the importing job. + duplicateEntityCounters.get(counterKey).inc(); + return; + } + if (entities.size() == 1) { if (isSpecialCaseProberEntity(entities.get(0))) { return;