Add PollMapreduceAction

This is the first in a series of CLs containing code from an old CL of Dai's that had never been completed, which compares zone data between Datastore and DNS. I had written a script to do this by calling two nomulus commands, but maybe it can be done directly in Java, which would be convenient.

This CL is just the plumbing to check on the status of a Mapreduce. We will need this to know that we can proceed with the next step of comparing the output to the DNS data.

Cloned from CL 134295050 by 'g4 patch'.
Original change by dxy@dxy:zoneman-reader:1939:citc on 2016/09/26 10:34:22.

Add a command for comparing zone data between DNS and datastore

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=167188979

java/google/registry/env/common/tools/WEB-INF/web.xml

@@ -128,6 +128,11 @@
   </servlet-mapping>
 
 
+  <servlet-mapping>
+    <servlet-name>tools-servlet</servlet-name>
+    <url-pattern>/_dr/task/pollMapreduce</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

java/google/registry/module/tools/ToolsRequestComponent.java

@@ -42,6 +42,7 @@ import google.registry.tools.server.ListPremiumListsAction;
 import google.registry.tools.server.ListRegistrarsAction;
 import google.registry.tools.server.ListReservedListsAction;
 import google.registry.tools.server.ListTldsAction;
+import google.registry.tools.server.PollMapreduceAction;
 import google.registry.tools.server.RefreshDnsForAllDomainsAction;
 import google.registry.tools.server.ResaveAllEppResourcesAction;
 import google.registry.tools.server.ToolsServerModule;
@@ -77,6 +78,7 @@ interface ToolsRequestComponent {
   ListReservedListsAction listReservedListsAction();
   ListTldsAction listTldsAction();
   LoadTestAction loadTestAction();
+  PollMapreduceAction pollMapReduceAction();
   PublishDetailReportAction publishDetailReportAction();
   RefreshDnsForAllDomainsAction refreshDnsForAllDomainsAction();
   ResaveAllEppResourcesAction resaveAllEppResourcesAction();

java/google/registry/tools/server/BUILD

@@ -25,6 +25,7 @@ java_library(
         "@com_google_appengine_api_1_0_sdk",
         "@com_google_appengine_tools_appengine_gcs_client",
         "@com_google_appengine_tools_appengine_mapreduce",
+        "@com_google_appengine_tools_appengine_pipeline",
         "@com_google_code_findbugs_jsr305",
         "@com_google_dagger",
         "@com_google_guava",

java/google/registry/tools/server/PollMapreduceAction.java

@@ -0,0 +1,58 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.server;
+
+import static com.google.appengine.tools.pipeline.PipelineServiceFactory.newPipelineService;
+import static google.registry.request.Action.Method.POST;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.appengine.tools.mapreduce.MapReduceResult;
+import com.google.appengine.tools.pipeline.JobInfo;
+import com.google.appengine.tools.pipeline.NoSuchObjectException;
+import com.google.common.collect.ImmutableMap;
+import google.registry.request.Action;
+import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.request.JsonResponse;
+import google.registry.request.Parameter;
+import google.registry.request.auth.Auth;
+import javax.inject.Inject;
+
+/** Action to poll the status of a mapreduce job. */
+@Action(path = PollMapreduceAction.PATH, method = POST, auth = Auth.AUTH_INTERNAL_ONLY)
+public class PollMapreduceAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/pollMapreduce";
+
+  @Inject @Parameter("jobId") String jobId;
+  @Inject JsonResponse response;
+  @Inject PollMapreduceAction() {}
+
+  @Override
+  public void run() {
+    JobInfo jobInfo;
+    try {
+      jobInfo = newPipelineService().getJobInfo(jobId);
+    } catch (NoSuchObjectException e) {
+      throw new InternalServerErrorException("Job not found: " + e);
+    }
+    ImmutableMap.Builder<String, String> json = new ImmutableMap.Builder<>();
+    json.put("state", jobInfo.getJobState().toString());
+    if (jobInfo.getJobState() == JobInfo.State.COMPLETED_SUCCESSFULLY) {
+      json.put("output", ((MapReduceResult<?>) jobInfo.getOutput()).getOutputResult().toString());
+    }
+    response.setPayload(json.build());
+    response.setStatus(SC_OK);
+  }
+}

java/google/registry/tools/server/ToolsServerModule.java

@@ -95,4 +95,10 @@ public class ToolsServerModule {
   static String provideRawKeys(HttpServletRequest req) {
     return extractRequiredParameter(req, "rawKeys");
   }
+
+  @Provides
+  @Parameter("jobId")
+  String provideJobId(HttpServletRequest req) {
+    return extractRequiredParameter(req, "jobId");
+  }
 }

javatests/google/registry/module/tools/testdata/tools_routing.txt

@@ -16,6 +16,7 @@ PATH                              CLASS                         METHODS  OK AUTH
 /_dr/task/generateZoneFiles       GenerateZoneFilesAction       POST     n  INTERNAL,API APP ADMIN
 /_dr/task/killAllCommitLogs       KillAllCommitLogsAction       POST     n  INTERNAL     APP IGNORED
 /_dr/task/killAllEppResources     KillAllEppResourcesAction     POST     n  INTERNAL     APP IGNORED
+/_dr/task/pollMapreduce           PollMapreduceAction           POST     n  INTERNAL     APP IGNORED
 /_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET      n  INTERNAL,API APP ADMIN
 /_dr/task/resaveAllEppResources   ResaveAllEppResourcesAction   GET      n  INTERNAL,API APP ADMIN
 /_dr/task/restoreCommitLogs       RestoreCommitLogsAction       POST     y  INTERNAL,API APP ADMIN

javatests/google/registry/tools/server/BUILD

@@ -26,6 +26,7 @@ java_library(
         "@com_google_appengine_api_1_0_sdk//:testonly",
         "@com_google_appengine_tools_appengine_gcs_client",
         "@com_google_appengine_tools_appengine_mapreduce",
+        "@com_google_appengine_tools_appengine_pipeline",
         "@com_google_guava",
         "@com_google_re2j",
         "@com_google_truth",

javatests/google/registry/tools/server/PollMapreduceActionTest.java

@@ -0,0 +1,91 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.server;
+
+import static com.google.appengine.tools.pipeline.PipelineServiceFactory.newPipelineService;
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.appengine.tools.mapreduce.Counters;
+import com.google.appengine.tools.mapreduce.MapReduceResult;
+import com.google.appengine.tools.pipeline.Job0;
+import com.google.appengine.tools.pipeline.JobInfo.State;
+import com.google.appengine.tools.pipeline.Value;
+import google.registry.testing.FakeJsonResponse;
+import google.registry.testing.mapreduce.MapreduceTestCase;
+import java.util.Map;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link PollMapreduceAction}.*/
+@RunWith(JUnit4.class)
+public class PollMapreduceActionTest extends MapreduceTestCase<PollMapreduceAction> {
+
+  @Before
+  public void init() throws Exception {
+    action = new PollMapreduceAction();
+  }
+
+  @Test
+  public void testPollUntilSuccess() throws Exception {
+    action.jobId = newPipelineService().startNewPipeline(new SuccessfulJob());
+    assertThat(poll()).containsExactly("state", State.RUNNING.toString());
+    executeTasksUntilEmpty("default");
+    assertThat(poll()).containsExactly(
+        "state", State.COMPLETED_SUCCESSFULLY.toString(), "output", "foobar");
+  }
+
+  @Test
+  public void testPollUntilFailure() throws Exception {
+    action.jobId = newPipelineService().startNewPipeline(new ThrowingJob());
+    assertThat(poll()).containsExactly("state", State.RUNNING.toString());
+    executeTasksUntilEmpty("default");
+    assertThat(poll()).containsExactly("state", State.STOPPED_BY_ERROR.toString());
+  }
+
+  Map<String, ?> poll() {
+    action.response = new FakeJsonResponse();
+    action.run();
+    return ((FakeJsonResponse) action.response).getResponseMap();
+  }
+
+  /** A job that returns a string. */
+  static class SuccessfulJob extends Job0<MapReduceResult<String>> {
+    @Override
+    public Value<MapReduceResult<String>> run() {
+      MapReduceResult<String> result =
+          new MapReduceResult<String>() {
+            @Override
+            public Counters getCounters() {
+              throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public String getOutputResult() {
+              return "foobar";
+            }};
+      return immediate(result);
+    }
+  }
+
+  /** A job that throws a RunTimeException when run. */
+  static class ThrowingJob extends Job0<Void> {
+    @Override
+    public Value<Void> run() {
+      throw new RuntimeException("expected");
+    }
+  }
+}