Import code from internal repository to git

java/com/google/domain/registry/tools/mapreduce/BUILD

@@ -0,0 +1,30 @@
+package(
+    default_visibility = ["//java/com/google/domain/registry:registry_project"],
+)
+
+
+java_library(
+    name = "mapreduce",
+    srcs = glob(["*.java"]),
+    deps = [
+        "//java/com/google/common/annotations",
+        "//java/com/google/common/base",
+        "//java/com/google/common/collect",
+        "//java/com/google/common/net",
+        "//java/com/google/common/primitives",
+        "//java/com/google/common/util/concurrent",
+        "//java/com/google/domain/registry/flows",
+        "//java/com/google/domain/registry/mapreduce",
+        "//java/com/google/domain/registry/model",
+        "//java/com/google/domain/registry/request",
+        "//java/com/google/domain/registry/util",
+        "//third_party/java/appengine_mapreduce2:appengine_mapreduce",
+        "//third_party/java/dagger",
+        "//third_party/java/jcommander",
+        "//third_party/java/joda_money",
+        "//third_party/java/joda_time",
+        "//third_party/java/jsr305_annotations",
+        "//third_party/java/jsr330_inject",
+        "//third_party/java/objectify:objectify-v4_1",
+    ],
+)

java/com/google/domain/registry/tools/mapreduce/DeleteProberDataAction.java

@@ -0,0 +1,171 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.domain.registry.tools.mapreduce;
+
+import static com.google.common.base.Verify.verifyNotNull;
+import static com.google.domain.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
+import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
+import static com.google.domain.registry.model.registry.Registries.getTldsOfType;
+
+import com.google.appengine.tools.mapreduce.Mapper;
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.base.Splitter;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.domain.registry.mapreduce.EppResourceInputs;
+import com.google.domain.registry.mapreduce.MapreduceRunner;
+import com.google.domain.registry.model.domain.DomainApplication;
+import com.google.domain.registry.model.domain.DomainBase;
+import com.google.domain.registry.model.index.EppResourceIndex;
+import com.google.domain.registry.model.index.ForeignKeyIndex;
+import com.google.domain.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
+import com.google.domain.registry.model.registry.Registry;
+import com.google.domain.registry.model.registry.Registry.TldType;
+import com.google.domain.registry.request.Action;
+import com.google.domain.registry.request.Parameter;
+import com.google.domain.registry.request.Response;
+import com.google.domain.registry.util.FormattingLogger;
+import com.google.domain.registry.util.PipelineUtils;
+
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.Work;
+
+import java.util.List;
+
+import javax.inject.Inject;
+
+/**
+ * Deletes all prober DomainResources and their subordinate history entries, poll messages, and
+ * billing events, along with their ForeignKeyDomainIndex and EppResourceIndex entities.
+ *
+ * <p>See: https://www.youtube.com/watch?v=xuuv0syoHnM
+ */
+@Action(path = "/_dr/task/deleteProberData")
+public class DeleteProberDataAction implements Runnable {
+
+  private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+  @Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
+  @Inject MapreduceRunner mrRunner;
+  @Inject Response response;
+  @Inject DeleteProberDataAction() {}
+
+  @Override
+  public void run() {
+    response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner
+        .setJobName("Delete prober data")
+        // TODO(b/27309488): maybe move this to the backend module.
+        .setModuleName("tools")
+        .runMapOnly(
+            new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun),
+            ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))));
+  }
+
+  private static ImmutableSet<String> getProberRoidSuffixes() {
+    return FluentIterable.from(getTldsOfType(TldType.TEST))
+        .filter(new Predicate<String>() {
+          @Override
+          public boolean apply(String tld) {
+            // Extra sanity check to prevent us from nuking prod data if a real TLD accidentally
+            // gets set to type TEST.
+            return tld.endsWith(".test");
+          }})
+        .transform(
+            new Function<String, String>() {
+              @Override
+              public String apply(String tld) {
+                return Registry.get(tld).getRoidSuffix();
+              }})
+        .toSet();
+  }
+
+  /** Provides the map method that runs for each existing DomainBase entity. */
+  public static class DeleteProberDataMapper extends Mapper<Key<DomainBase>, Void, Void> {
+
+    private static final long serialVersionUID = 1737761271804180412L;
+
+    private final ImmutableSet<String> proberRoidSuffixes;
+    private final Boolean isDryRun;
+
+    public DeleteProberDataMapper(ImmutableSet<String> proberRoidSuffixes, Boolean isDryRun) {
+      this.proberRoidSuffixes = proberRoidSuffixes;
+      this.isDryRun = isDryRun;
+    }
+
+    @Override
+    public final void map(Key<DomainBase> key) {
+      try {
+        String roidSuffix = Iterables.getLast(Splitter.on('-').split(key.getName()));
+        if (proberRoidSuffixes.contains(roidSuffix)) {
+          deleteDomain(key);
+        } else {
+          getContext().incrementCounter(String.format("skipped, non-prober data"));
+        }
+      } catch (Throwable t) {
+        logger.severefmt(t, "Error while deleting prober data for key %s", key);
+        getContext().incrementCounter(String.format("error, kind %s", key.getKind()));
+      }
+    }
+
+    private void deleteDomain(final Key<DomainBase> domainKey) {
+      final DomainBase domain = ofy().load().key(domainKey).now();
+      if (domain == null) {
+        // Depending on how stale Datastore indexes are, we can get keys to resources that are
+        // already deleted (e.g. by a recent previous invocation of this mapreduce). So ignore them.
+        getContext().incrementCounter("already deleted");
+        return;
+      }
+      if (domain instanceof DomainApplication) {
+        // Cover the case where we somehow have a domain application with a prober ROID suffix.
+        getContext().incrementCounter("skipped, domain application");
+        return;
+      }
+      if (domain.getFullyQualifiedDomainName().equals("nic." + domain.getTld())) {
+        getContext().incrementCounter("skipped, NIC domain");
+        return;
+      }
+      int dependentsDeleted = ofy().transact(new Work<Integer>() {
+        @Override
+        public Integer run() {
+          EppResourceIndex eppIndex = ofy().load().entity(EppResourceIndex.create(domainKey)).now();
+          verifyNotNull(eppIndex, "Missing EppResourceIndex for domain %s", domain);
+          ForeignKeyIndex<?> fki = ofy().load().key(ForeignKeyDomainIndex.createKey(domain)).now();
+          verifyNotNull(fki, "Missing ForeignKeyDomainIndex for domain %s", domain);
+          // This ancestor query selects all descendant HistoryEntries, BillingEvents, and
+          // PollMessages, as well as the domain itself.
+          List<Key<Object>> domainAndDependentKeys = ofy().load().ancestor(domainKey).keys().list();
+          if (isDryRun) {
+            logger.infofmt(
+                "Would delete the following entities: %s",
+                new ImmutableList.Builder<Object>()
+                    .add(fki)
+                    .add(eppIndex)
+                    .addAll(domainAndDependentKeys)
+                    .build());
+          } else {
+            ofy().deleteWithoutBackup().keys(domainAndDependentKeys);
+            ofy().deleteWithoutBackup().entities(eppIndex, fki);
+          }
+          return domainAndDependentKeys.size() - 1;
+        }
+      });
+      getContext().incrementCounter(String.format("deleted, kind %s", domainKey.getKind()));
+      getContext().incrementCounter("deleted, dependent keys", dependentsDeleted);
+    }
+  }
+}

java/com/google/domain/registry/tools/mapreduce/ResaveAllEppResourcesAction.java

@@ -0,0 +1,76 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.domain.registry.tools.mapreduce;
+
+import static com.google.domain.registry.mapreduce.EppResourceInputs.createEntityInput;
+import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
+import static com.google.domain.registry.util.PipelineUtils.createJobPath;
+
+import com.google.appengine.tools.mapreduce.Mapper;
+import com.google.common.collect.ImmutableList;
+import com.google.domain.registry.mapreduce.MapreduceAction;
+import com.google.domain.registry.mapreduce.MapreduceRunner;
+import com.google.domain.registry.model.EppResource;
+import com.google.domain.registry.request.Action;
+import com.google.domain.registry.request.Response;
+
+import com.googlecode.objectify.VoidWork;
+
+import javax.inject.Inject;
+
+/**
+ * A mapreduce that re-saves all EppResources without otherwise modifying them.
+ *
+ * <p>This is useful for completing data migrations on EppResource fields that are accomplished
+ * with @OnSave or @OnLoad annotations, and also guarantees that all EppResources will get fresh
+ * commit logs (for backup purposes).
+ */
+@Action(path = "/_dr/task/resaveAllEppResources")
+public class ResaveAllEppResourcesAction implements MapreduceAction {
+
+  @Inject MapreduceRunner mrRunner;
+  @Inject Response response;
+  @Inject ResaveAllEppResourcesAction() {}
+
+  @SuppressWarnings("unchecked")
+  @Override
+  public void run() {
+    response.sendJavaScriptRedirect(createJobPath(mrRunner
+        .setJobName("Re-save all EPP resources")
+        .setModuleName("backend")
+        .runMapOnly(
+            new ResaveAllEppResourcesActionMapper(),
+            ImmutableList.of(createEntityInput(EppResource.class)))));
+  }
+
+  /** Mapper to re-save all EPP resources. */
+  public static class ResaveAllEppResourcesActionMapper extends Mapper<EppResource, Void, Void> {
+
+    private static final long serialVersionUID = -7721628665138087001L;
+    public ResaveAllEppResourcesActionMapper() {}
+
+    @Override
+    public final void map(final EppResource resource) {
+      ofy().transact(new VoidWork() {
+        @Override
+        public void vrun() {
+          ofy().save().entity(resource).now();
+        }});
+      getContext().incrementCounter(
+          String.format("%s entities re-saved", resource.getClass().getSimpleName()));
+    }
+  }
+}
+