zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-08-12 04:29:39 +02:00
Code Issues Projects Releases Packages Wiki Activity

Count duplicates when comparing Databases (#1509)

Browse source 
* Count duplicates when comparing Databases

Cursors may have duplicates in Datastore if imported across projects.
Count them instead of throwing.
This commit is contained in:
Weimin Yu 2022-02-03 10:59:03 -05:00 • committed by GitHub
parent 86d227c748
commit 77600ba404
1 changed files with 10 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

11
core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java
View file

@ -104,6 +104,7 @@ final class ValidateSqlUtils {
private final HashMap<String, Counter> missingCounters = new HashMap<>(); private final HashMap<String, Counter> missingCounters = new HashMap<>();
private final HashMap<String, Counter> unequalCounters = new HashMap<>(); private final HashMap<String, Counter> unequalCounters = new HashMap<>();
private final HashMap<String, Counter> badEntityCounters = new HashMap<>(); private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
private final HashMap<String, Counter> duplicateEntityCounters = new HashMap<>();
private volatile boolean logPrinted = false; private volatile boolean logPrinted = false;
@ -120,6 +121,8 @@ final class ValidateSqlUtils {
counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey)); counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey)); unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey)); badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
duplicateEntityCounters.put(
counterKey, Metrics.counter("CompareDB", "Duplicate Entities:" + counterKey));
} }
/** /**
@ -158,12 +161,18 @@ final class ValidateSqlUtils {
ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue()); ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey()); verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
String counterKey = getCounterKey(entities.get(0).getClass()); String counterKey = getCounterKey(entities.get(0).getClass());
ensureCounterExists(counterKey); ensureCounterExists(counterKey);
totalCounters.get(counterKey).inc(); totalCounters.get(counterKey).inc();
if (entities.size() > 2) {
// Duplicates may happen with Cursors if imported across projects. Its key in Datastore, the
// id field, encodes the project name and is not fixed by the importing job.
duplicateEntityCounters.get(counterKey).inc();
return;
}
if (entities.size() == 1) { if (entities.size() == 1) {
if (isSpecialCaseProberEntity(entities.get(0))) { if (isSpecialCaseProberEntity(entities.get(0))) {
return; return;