zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-13 16:07:15 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add and fix mapreduce counters

Browse source 
Lessons from the first full run of the mapreduce.

Fixed the "old manifests still referenced" counter, since some of these
manifests could be new.

Added a debug "manifests incorrectly mapped multiple times" that is expected to never happen, but who knows? (this is a sort of sanity check for the mapreduce)

Added an "Epp resource revisions handled" that counts the total references, allowing for the possibility of the same manifest being referenced from multiple resources - this is expected to be equal to the map's "Epp resource revisions found" counter.

Added a "commit log manifests referenced multiple times" to see how often the same manifest is linked to from different epp resources.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=166846879
This commit is contained in:
guyben 2017-08-29 07:57:06 -07:00 • committed by Ben McIlwain
parent 63fdb506df
commit ccc51daa9f
1 changed files with 17 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

20
java/google/registry/backup/DeleteOldCommitLogsAction.java
View file

@ -21,6 +21,7 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.FormattingLogger.getLoggerForCallerClass; import static google.registry.util.FormattingLogger.getLoggerForCallerClass;
import static google.registry.util.PipelineUtils.createJobPath; import static google.registry.util.PipelineUtils.createJobPath;
import static java.lang.Boolean.FALSE; import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.Reducer; import com.google.appengine.tools.mapreduce.Reducer;
@ -28,7 +29,7 @@ import com.google.appengine.tools.mapreduce.ReducerInput;
import com.google.auto.value.AutoValue; import com.google.auto.value.AutoValue;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableMultiset;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import com.googlecode.objectify.Work; import com.googlecode.objectify.Work;
import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryConfig.Config;
@ -204,8 +205,21 @@ public final class DeleteOldCommitLogsAction implements Runnable {
public void reduce( public void reduce(
final Key<CommitLogManifest> manifestKey, final Key<CommitLogManifest> manifestKey,
ReducerInput<Boolean> canDeleteVerdicts) { ReducerInput<Boolean> canDeleteVerdicts) {
if (ImmutableSet.copyOf(canDeleteVerdicts).contains(FALSE)) { ImmutableMultiset<Boolean> canDeleteMultiset = ImmutableMultiset.copyOf(canDeleteVerdicts);
getContext().incrementCounter("old commit log manifests still referenced"); if (canDeleteMultiset.count(TRUE) > 1) {
getContext().incrementCounter("commit log manifests incorrectly mapped multiple times");
}
if (canDeleteMultiset.count(FALSE) > 1) {
getContext().incrementCounter("commit log manifests referenced multiple times");
}
if (canDeleteMultiset.contains(FALSE)) {
getContext().incrementCounter(
canDeleteMultiset.contains(TRUE)
? "old commit log manifests still referenced"
: "new (or nonexistent) commit log manifests referenced");
getContext().incrementCounter(
"EPP resource revisions handled",
canDeleteMultiset.count(FALSE));
return; return;
} }