diff --git a/java/google/registry/backup/DeleteOldCommitLogsAction.java b/java/google/registry/backup/DeleteOldCommitLogsAction.java index fde9a88ff..3efe87876 100644 --- a/java/google/registry/backup/DeleteOldCommitLogsAction.java +++ b/java/google/registry/backup/DeleteOldCommitLogsAction.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PipelineUtils.createJobPath; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; @@ -88,17 +87,18 @@ public final class DeleteOldCommitLogsAction implements Runnable { "Processing asynchronous deletion of unreferenced CommitLogManifests older than %s", deletionThreshold); - response.sendJavaScriptRedirect(createJobPath(mrRunner - .setJobName("Delete old commit logs") - .setModuleName("backend") - .setDefaultMapShards(NUM_MAP_SHARDS) - .setDefaultReduceShards(NUM_REDUCE_SHARDS) - .runMapreduce( - new DeleteOldCommitLogsMapper(deletionThreshold), - new DeleteOldCommitLogsReducer(deletionThreshold, isDryRun), - ImmutableList.of( - new CommitLogManifestInput(deletionThreshold), - EppResourceInputs.createKeyInput(EppResource.class))))); + mrRunner + .setJobName("Delete old commit logs") + .setModuleName("backend") + .setDefaultMapShards(NUM_MAP_SHARDS) + .setDefaultReduceShards(NUM_REDUCE_SHARDS) + .runMapreduce( + new DeleteOldCommitLogsMapper(deletionThreshold), + new DeleteOldCommitLogsReducer(deletionThreshold, isDryRun), + ImmutableList.of( + new CommitLogManifestInput(deletionThreshold), + EppResourceInputs.createKeyInput(EppResource.class))) + .sendLinkToMapreduceConsole(response); } /** diff --git a/java/google/registry/batch/DeleteContactsAndHostsAction.java b/java/google/registry/batch/DeleteContactsAndHostsAction.java index e62245f9d..704badfae 100644 --- a/java/google/registry/batch/DeleteContactsAndHostsAction.java +++ b/java/google/registry/batch/DeleteContactsAndHostsAction.java @@ -40,7 +40,6 @@ import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE_F import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE; import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE_FAILURE; import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED; -import static google.registry.util.PipelineUtils.createJobPath; import static java.math.RoundingMode.CEILING; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -215,20 +214,18 @@ public class DeleteContactsAndHostsAction implements Runnable { try { int numReducers = Math.min(MAX_REDUCE_SHARDS, divide(deletionRequests.size(), DELETES_PER_SHARD, CEILING)); - response.sendJavaScriptRedirect( - createJobPath( - mrRunner - .setJobName("Check for EPP resource references and then delete") - .setModuleName("backend") - .setDefaultReduceShards(numReducers) - .runMapreduce( - new DeleteContactsAndHostsMapper(deletionRequests), - new DeleteEppResourceReducer(), - ImmutableList.of( - // Add an extra shard that maps over a null domain. See the mapper code - // for why. - new NullInput<>(), EppResourceInputs.createEntityInput(DomainBase.class)), - new UnlockerOutput(lock.get())))); + mrRunner + .setJobName("Check for EPP resource references and then delete") + .setModuleName("backend") + .setDefaultReduceShards(numReducers) + .runMapreduce( + new DeleteContactsAndHostsMapper(deletionRequests), + new DeleteEppResourceReducer(), + ImmutableList.of( + // Add an extra shard that maps over a null domain. See the mapper code for why. + new NullInput<>(), EppResourceInputs.createEntityInput(DomainBase.class)), + new UnlockerOutput(lock.get())) + .sendLinkToMapreduceConsole(response); } catch (Throwable t) { logRespondAndUnlock(SEVERE, "Error starting mapreduce to delete contacts/hosts.", lock); } diff --git a/java/google/registry/batch/DeleteLoadTestDataAction.java b/java/google/registry/batch/DeleteLoadTestDataAction.java index 3ceb59810..bfdb159a0 100644 --- a/java/google/registry/batch/DeleteLoadTestDataAction.java +++ b/java/google/registry/batch/DeleteLoadTestDataAction.java @@ -37,7 +37,6 @@ import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; -import google.registry.util.PipelineUtils; import java.util.List; import javax.inject.Inject; @@ -83,16 +82,14 @@ public class DeleteLoadTestDataAction implements Runnable { checkState( registryEnvironment != PRODUCTION, "This mapreduce is not safe to run on PRODUCTION."); - response.sendJavaScriptRedirect( - PipelineUtils.createJobPath( - mrRunner - .setJobName("Delete load test data") - .setModuleName("backend") - .runMapOnly( - new DeleteLoadTestDataMapper(isDryRun), - ImmutableList.of( - createEntityInput(ContactResource.class), - createEntityInput(HostResource.class))))); + mrRunner + .setJobName("Delete load test data") + .setModuleName("backend") + .runMapOnly( + new DeleteLoadTestDataMapper(isDryRun), + ImmutableList.of( + createEntityInput(ContactResource.class), createEntityInput(HostResource.class))) + .sendLinkToMapreduceConsole(response); } /** Provides the map method that runs for each existing contact and host entity. */ diff --git a/java/google/registry/batch/DeleteProberDataAction.java b/java/google/registry/batch/DeleteProberDataAction.java index 5bfcbf38c..35386a526 100644 --- a/java/google/registry/batch/DeleteProberDataAction.java +++ b/java/google/registry/batch/DeleteProberDataAction.java @@ -52,7 +52,6 @@ import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; -import google.registry.util.PipelineUtils; import java.util.List; import javax.inject.Inject; import org.joda.time.DateTime; @@ -87,12 +86,13 @@ public class DeleteProberDataAction implements Runnable { checkState( !Strings.isNullOrEmpty(registryAdminClientId), "Registry admin client ID must be configured for prober data deletion to work"); - response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner + mrRunner .setJobName("Delete prober data") .setModuleName("backend") .runMapOnly( new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId), - ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class))))); + ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class))) + .sendLinkToMapreduceConsole(response); } private ImmutableSet getProberRoidSuffixes() { diff --git a/java/google/registry/batch/ExpandRecurringBillingEventsAction.java b/java/google/registry/batch/ExpandRecurringBillingEventsAction.java index 443050c89..0490f411c 100644 --- a/java/google/registry/batch/ExpandRecurringBillingEventsAction.java +++ b/java/google/registry/batch/ExpandRecurringBillingEventsAction.java @@ -28,7 +28,6 @@ import static google.registry.util.CollectionUtils.union; import static google.registry.util.DateTimeUtils.START_OF_TIME; import static google.registry.util.DateTimeUtils.earliestOf; import static google.registry.util.DomainNameUtils.getTldFromDomainName; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.mapreduce.Mapper; import com.google.appengine.tools.mapreduce.Reducer; @@ -102,7 +101,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable { logger.atInfo().log( "Running Recurring billing event expansion for billing time range [%s, %s).", cursorTime, executeTime); - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Expand Recurring billing events into synthetic OneTime events.") .setModuleName("backend") .runMapreduce( @@ -112,7 +111,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable { ImmutableList.of( new NullInput<>(), createChildEntityInput( - ImmutableSet.of(DomainResource.class), ImmutableSet.of(Recurring.class)))))); + ImmutableSet.of(DomainResource.class), ImmutableSet.of(Recurring.class)))) + .sendLinkToMapreduceConsole(response); } /** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */ diff --git a/java/google/registry/batch/RefreshDnsOnHostRenameAction.java b/java/google/registry/batch/RefreshDnsOnHostRenameAction.java index 55b593641..8858ee19a 100644 --- a/java/google/registry/batch/RefreshDnsOnHostRenameAction.java +++ b/java/google/registry/batch/RefreshDnsOnHostRenameAction.java @@ -27,7 +27,6 @@ import static google.registry.model.EppResourceUtils.isActive; import static google.registry.model.EppResourceUtils.isDeleted; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.util.DateTimeUtils.latestOf; -import static google.registry.util.PipelineUtils.createJobPath; import static java.util.concurrent.TimeUnit.DAYS; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.logging.Level.INFO; @@ -159,18 +158,16 @@ public class RefreshDnsOnHostRenameAction implements Runnable { private void runMapreduce(ImmutableList refreshRequests, Optional lock) { try { - response.sendJavaScriptRedirect( - createJobPath( - mrRunner - .setJobName("Enqueue DNS refreshes for domains referencing renamed hosts") - .setModuleName("backend") - .setDefaultReduceShards(1) - .runMapreduce( - new RefreshDnsOnHostRenameMapper(refreshRequests, retrier), - new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier), - // Add an extra NullInput so that the reducer always fires exactly once. - ImmutableList.of( - new NullInput<>(), createEntityInput(DomainResource.class))))); + mrRunner + .setJobName("Enqueue DNS refreshes for domains referencing renamed hosts") + .setModuleName("backend") + .setDefaultReduceShards(1) + .runMapreduce( + new RefreshDnsOnHostRenameMapper(refreshRequests, retrier), + new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier), + // Add an extra NullInput so that the reducer always fires exactly once. + ImmutableList.of(new NullInput<>(), createEntityInput(DomainResource.class))) + .sendLinkToMapreduceConsole(response); } catch (Throwable t) { logRespondAndUnlock( SEVERE, "Error starting mapreduce to refresh DNS for renamed hosts.", lock); diff --git a/java/google/registry/batch/ResaveAllEppResourcesAction.java b/java/google/registry/batch/ResaveAllEppResourcesAction.java index 57f2e5481..8ce341837 100644 --- a/java/google/registry/batch/ResaveAllEppResourcesAction.java +++ b/java/google/registry/batch/ResaveAllEppResourcesAction.java @@ -15,7 +15,6 @@ package google.registry.batch; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.mapreduce.Mapper; import com.google.common.collect.ImmutableList; @@ -52,12 +51,13 @@ public class ResaveAllEppResourcesAction implements Runnable { @Override public void run() { - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Re-save all EPP resources") .setModuleName("backend") .runMapOnly( new ResaveAllEppResourcesActionMapper(), - ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class))))); + ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class))) + .sendLinkToMapreduceConsole(response); } /** Mapper to re-save all EPP resources. */ diff --git a/java/google/registry/export/ExportDomainListsAction.java b/java/google/registry/export/ExportDomainListsAction.java index c42afc04a..1ed45fcd1 100644 --- a/java/google/registry/export/ExportDomainListsAction.java +++ b/java/google/registry/export/ExportDomainListsAction.java @@ -20,7 +20,6 @@ import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInp import static google.registry.model.EppResourceUtils.isActive; import static google.registry.model.registry.Registries.getTldsOfType; import static google.registry.request.Action.Method.POST; -import static google.registry.util.PipelineUtils.createJobPath; import static java.nio.charset.StandardCharsets.UTF_8; import static org.joda.time.DateTimeZone.UTC; @@ -78,14 +77,15 @@ public class ExportDomainListsAction implements Runnable { public void run() { ImmutableSet realTlds = getTldsOfType(TldType.REAL); logger.atInfo().log("Exporting domain lists for tlds %s", realTlds); - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Export domain lists") .setModuleName("backend") .setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS)) .runMapreduce( new ExportDomainListsMapper(DateTime.now(UTC), realTlds), new ExportDomainListsReducer(gcsBucket, gcsBufferSize), - ImmutableList.of(createEntityInput(DomainResource.class))))); + ImmutableList.of(createEntityInput(DomainResource.class))) + .sendLinkToMapreduceConsole(response); } static class ExportDomainListsMapper extends Mapper { diff --git a/java/google/registry/mapreduce/MapreduceRunner.java b/java/google/registry/mapreduce/MapreduceRunner.java index 012522a99..39c07db21 100644 --- a/java/google/registry/mapreduce/MapreduceRunner.java +++ b/java/google/registry/mapreduce/MapreduceRunner.java @@ -35,7 +35,8 @@ import com.google.appengine.tools.pipeline.JobSetting; import com.google.common.flogger.FluentLogger; import google.registry.mapreduce.inputs.ConcatenatingInput; import google.registry.request.Parameter; -import google.registry.util.PipelineUtils; +import google.registry.request.Response; +import google.registry.util.AppEngineServiceUtils; import java.io.Serializable; import java.util.Optional; import javax.inject.Inject; @@ -58,8 +59,12 @@ public class MapreduceRunner { private static final String BASE_URL = "/_dr/mapreduce/"; private static final String QUEUE_NAME = "mapreduce"; + private static final String MAPREDUCE_CONSOLE_LINK_FORMAT = + "Mapreduce console: https://%s/_ah/pipeline/status.html?root=%s"; + private final Optional httpParamMapShards; private final Optional httpParamReduceShards; + private final AppEngineServiceUtils appEngineServiceUtils; // Default to 3 minutes since many slices will contain Datastore queries that time out at 4:30. private Duration sliceDuration = Duration.standardMinutes(3); @@ -87,9 +92,11 @@ public class MapreduceRunner { @Inject public MapreduceRunner( @Parameter(PARAM_MAP_SHARDS) Optional mapShards, - @Parameter(PARAM_REDUCE_SHARDS) Optional reduceShards) { + @Parameter(PARAM_REDUCE_SHARDS) Optional reduceShards, + AppEngineServiceUtils appEngineServiceUtils) { this.httpParamMapShards = mapShards; this.httpParamReduceShards = reduceShards; + this.appEngineServiceUtils = appEngineServiceUtils; } /** Set the max time to run a slice before serializing; defaults to 3 minutes. */ @@ -160,15 +167,13 @@ public class MapreduceRunner { * all work will be accomplished via side effects during the map phase. * * @see #createMapOnlyJob for creating and running a map-only mapreduce as part of a pipeline - * * @param mapper instance of a mapper class * @param inputs input sources for the mapper * @param mapper input type * @return the job id */ - public String runMapOnly( - Mapper mapper, - Iterable> inputs) { + public MapreduceRunnerResult runMapOnly( + Mapper mapper, Iterable> inputs) { return runAsPipeline(createMapOnlyJob(mapper, new NoOutput(), inputs)); } @@ -220,7 +225,6 @@ public class MapreduceRunner { * all work will be accomplished via side effects during the map or reduce phases. * * @see #createMapreduceJob for creating and running a mapreduce as part of a pipeline - * * @param mapper instance of a mapper class * @param reducer instance of a reducer class * @param inputs input sources for the mapper @@ -229,10 +233,11 @@ public class MapreduceRunner { * @param emitted value type * @return the job id */ - public final String runMapreduce( - Mapper mapper, - Reducer reducer, - Iterable> inputs) { + public final + MapreduceRunnerResult runMapreduce( + Mapper mapper, + Reducer reducer, + Iterable> inputs) { return runMapreduce(mapper, reducer, inputs, new NoOutput()); } @@ -240,7 +245,6 @@ public class MapreduceRunner { * Kick off a mapreduce job with specified Output handler. * * @see #createMapreduceJob for creating and running a mapreduce as part of a pipeline - * * @param mapper instance of a mapper class * @param reducer instance of a reducer class * @param inputs input sources for the mapper @@ -251,11 +255,12 @@ public class MapreduceRunner { * @param return value of output * @return the job id */ - public final String runMapreduce( - Mapper mapper, - Reducer reducer, - Iterable> inputs, - Output output) { + public final + MapreduceRunnerResult runMapreduce( + Mapper mapper, + Reducer reducer, + Iterable> inputs, + Output output) { return runAsPipeline(createMapreduceJob(mapper, reducer, inputs, output)); } @@ -266,14 +271,41 @@ public class MapreduceRunner { checkArgumentNotNull(mapper, "mapper"); } - private String runAsPipeline(Job0 job) { - String jobId = newPipelineService().startNewPipeline( - job, - new JobSetting.OnModule(moduleName), - new JobSetting.OnQueue(QUEUE_NAME)); + private MapreduceRunnerResult runAsPipeline(Job0 job) { + String jobId = + newPipelineService() + .startNewPipeline( + job, new JobSetting.OnModule(moduleName), new JobSetting.OnQueue(QUEUE_NAME)); logger.atInfo().log( "Started '%s' %s job: %s", - jobName, job instanceof MapJob ? "map" : "mapreduce", PipelineUtils.createJobPath(jobId)); - return jobId; + jobName, job instanceof MapJob ? "map" : "mapreduce", renderMapreduceConsoleLink(jobId)); + return new MapreduceRunnerResult(jobId); + } + + private String renderMapreduceConsoleLink(String jobId) { + return String.format( + MAPREDUCE_CONSOLE_LINK_FORMAT, appEngineServiceUtils.getServiceHostname("backend"), jobId); + } + + /** + * Class representing the result of kicking off a mapreduce. + * + *

This is used to send a link to the mapreduce console. + */ + public class MapreduceRunnerResult { + + private final String jobId; + + private MapreduceRunnerResult(String jobId) { + this.jobId = jobId; + } + + public void sendLinkToMapreduceConsole(Response response) { + response.setPayload(getLinkToMapreduceConsole()); + } + + public String getLinkToMapreduceConsole() { + return renderMapreduceConsoleLink(jobId); + } } } diff --git a/java/google/registry/rde/RdeStagingAction.java b/java/google/registry/rde/RdeStagingAction.java index fed0bc3a2..5edc658bc 100644 --- a/java/google/registry/rde/RdeStagingAction.java +++ b/java/google/registry/rde/RdeStagingAction.java @@ -16,7 +16,6 @@ package google.registry.rde; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.POST; -import static google.registry.util.PipelineUtils.createJobPath; import static google.registry.xml.ValidationMode.LENIENT; import static google.registry.xml.ValidationMode.STRICT; import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; @@ -228,7 +227,7 @@ public final class RdeStagingAction implements Runnable { } RdeStagingMapper mapper = new RdeStagingMapper(lenient ? LENIENT : STRICT, pendings); - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Stage escrow deposits for all TLDs") .setModuleName("backend") .setDefaultReduceShards(pendings.size()) @@ -237,8 +236,8 @@ public final class RdeStagingAction implements Runnable { reducer, ImmutableList.of( // Add an extra shard that maps over a null resource. See the mapper code for why. - new NullInput<>(), - EppResourceInputs.createEntityInput(EppResource.class))))); + new NullInput<>(), EppResourceInputs.createEntityInput(EppResource.class))) + .sendLinkToMapreduceConsole(response); } private ImmutableSetMultimap getStandardPendingDeposits() { diff --git a/java/google/registry/rde/imports/RdeContactImportAction.java b/java/google/registry/rde/imports/RdeContactImportAction.java index bcde3caa7..e237e3129 100644 --- a/java/google/registry/rde/imports/RdeContactImportAction.java +++ b/java/google/registry/rde/imports/RdeContactImportAction.java @@ -16,7 +16,6 @@ package google.registry.rde.imports; import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.GcsServiceFactory; @@ -77,12 +76,11 @@ public class RdeContactImportAction implements Runnable { @Override public void run() { - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Import contacts from escrow file") .setModuleName("backend") - .runMapOnly( - createMapper(), - ImmutableList.of(createInput())))); + .runMapOnly(createMapper(), ImmutableList.of(createInput())) + .sendLinkToMapreduceConsole(response); } /** diff --git a/java/google/registry/rde/imports/RdeDomainImportAction.java b/java/google/registry/rde/imports/RdeDomainImportAction.java index 97043d9d8..3dda71d65 100644 --- a/java/google/registry/rde/imports/RdeDomainImportAction.java +++ b/java/google/registry/rde/imports/RdeDomainImportAction.java @@ -25,7 +25,6 @@ import static google.registry.rde.imports.RdeImportUtils.createAutoRenewBillingE import static google.registry.rde.imports.RdeImportUtils.createAutoRenewPollMessageForDomainImport; import static google.registry.rde.imports.RdeImportUtils.createHistoryEntryForDomainImport; import static google.registry.rde.imports.RdeImportsModule.PATH; -import static google.registry.util.PipelineUtils.createJobPath; import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import com.google.appengine.tools.cloudstorage.GcsService; @@ -107,12 +106,11 @@ public class RdeDomainImportAction implements Runnable { logger.atInfo().log( "Launching domains import mapreduce: bucket=%s, filename=%s", this.importBucketName, this.importFileName); - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Import domains from escrow file") .setModuleName("backend") - .runMapOnly( - createMapper(), - ImmutableList.of(createInput())))); + .runMapOnly(createMapper(), ImmutableList.of(createInput())) + .sendLinkToMapreduceConsole(response); } /** diff --git a/java/google/registry/rde/imports/RdeHostImportAction.java b/java/google/registry/rde/imports/RdeHostImportAction.java index 2e593e4df..53e2fa59f 100644 --- a/java/google/registry/rde/imports/RdeHostImportAction.java +++ b/java/google/registry/rde/imports/RdeHostImportAction.java @@ -16,7 +16,6 @@ package google.registry.rde.imports; import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.GcsServiceFactory; @@ -77,12 +76,13 @@ public class RdeHostImportAction implements Runnable { @Override public void run() { - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Import hosts from escrow file") .setModuleName("backend") .runMapOnly( new RdeHostImportMapper(importBucketName), - ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName))))); + ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName))) + .sendLinkToMapreduceConsole(response); } /** Mapper to import hosts from an escrow file. */ diff --git a/java/google/registry/rde/imports/RdeHostLinkAction.java b/java/google/registry/rde/imports/RdeHostLinkAction.java index 8b81f826f..6abe732c4 100644 --- a/java/google/registry/rde/imports/RdeHostLinkAction.java +++ b/java/google/registry/rde/imports/RdeHostLinkAction.java @@ -19,7 +19,6 @@ import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS; import static google.registry.model.EppResourceUtils.loadByForeignKey; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.model.registry.Registries.findTldForName; -import static google.registry.util.PipelineUtils.createJobPath; import static java.util.stream.Collectors.joining; import com.google.appengine.tools.mapreduce.Mapper; @@ -86,12 +85,13 @@ public class RdeHostLinkAction implements Runnable { @Override public void run() { - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Link hosts from escrow file") .setModuleName("backend") .runMapOnly( new RdeHostPostImportMapper(), - ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName))))); + ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName))) + .sendLinkToMapreduceConsole(response); } /** Mapper to link hosts from an escrow file to their superordinate domains. */ diff --git a/java/google/registry/request/Response.java b/java/google/registry/request/Response.java index 53490eb95..b0fcc147a 100644 --- a/java/google/registry/request/Response.java +++ b/java/google/registry/request/Response.java @@ -51,12 +51,4 @@ public interface Response { * @see HttpServletResponse#setDateHeader(String, long) */ void setDateHeader(String header, DateTime timestamp); - - /** - * Sends a JavaScript redirect HTTP response. - * - * GAE handles a HTTP 302 status as an error, so using this is helpful for responses that might - * sometimes be consumed by GAE code, since it performs a redirect while also returning HTTP 200. - */ - void sendJavaScriptRedirect(String redirectUrl); } diff --git a/java/google/registry/request/ResponseImpl.java b/java/google/registry/request/ResponseImpl.java index 884982061..0e14218a5 100644 --- a/java/google/registry/request/ResponseImpl.java +++ b/java/google/registry/request/ResponseImpl.java @@ -23,10 +23,6 @@ import org.joda.time.DateTime; /** HTTP response object. */ public final class ResponseImpl implements Response { - /** Code for a JavaScript redirect. */ - private static final String REDIRECT_PAYLOAD_FORMAT = - "%1$s"; - private final HttpServletResponse rsp; @Inject @@ -62,9 +58,4 @@ public final class ResponseImpl implements Response { public void setDateHeader(String header, DateTime timestamp) { rsp.setDateHeader(header, timestamp.getMillis()); } - - @Override - public void sendJavaScriptRedirect(String redirectUrl) { - setPayload(String.format(REDIRECT_PAYLOAD_FORMAT, redirectUrl)); - } } diff --git a/java/google/registry/tools/GenerateZoneFilesCommand.java b/java/google/registry/tools/GenerateZoneFilesCommand.java index 19351bc8f..ce9d92826 100644 --- a/java/google/registry/tools/GenerateZoneFilesCommand.java +++ b/java/google/registry/tools/GenerateZoneFilesCommand.java @@ -59,7 +59,7 @@ final class GenerateZoneFilesCommand implements CommandWithConnection, CommandWi "tlds", mainParameters, "exportTime", exportDate.toString()); Map response = connection.sendJson(GenerateZoneFilesAction.PATH, params); - System.out.printf("Job started at %s %s\n", connection.getServer(), response.get("jobPath")); + System.out.println(response.get("mapreduceConsoleLink")); System.out.println("Output files:"); @SuppressWarnings("unchecked") List filenames = (List) response.get("filenames"); diff --git a/java/google/registry/tools/server/GenerateZoneFilesAction.java b/java/google/registry/tools/server/GenerateZoneFilesAction.java index c1afdab0f..5f575f35d 100644 --- a/java/google/registry/tools/server/GenerateZoneFilesAction.java +++ b/java/google/registry/tools/server/GenerateZoneFilesAction.java @@ -22,7 +22,6 @@ import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInp import static google.registry.model.EppResourceUtils.loadAtPointInTime; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.request.Action.Method.POST; -import static google.registry.util.PipelineUtils.createJobPath; import static java.nio.charset.StandardCharsets.UTF_8; import static org.joda.time.DateTimeZone.UTC; @@ -132,17 +131,17 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA if (!exportTime.equals(exportTime.toDateTime(UTC).withTimeAtStartOfDay())) { throw new BadRequestException("Invalid export time: must be midnight UTC"); } - String jobId = mrRunner - .setJobName("Generate bind file stanzas") - .setModuleName("tools") - .setDefaultReduceShards(tlds.size()) - .runMapreduce( - new GenerateBindFileMapper( - tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl), - new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize), - ImmutableList.of( - new NullInput<>(), - createEntityInput(DomainResource.class))); + String mapreduceConsoleLink = + mrRunner + .setJobName("Generate bind file stanzas") + .setModuleName("tools") + .setDefaultReduceShards(tlds.size()) + .runMapreduce( + new GenerateBindFileMapper( + tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl), + new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize), + ImmutableList.of(new NullInput<>(), createEntityInput(DomainResource.class))) + .getLinkToMapreduceConsole(); ImmutableList filenames = tlds.stream() .map( @@ -151,7 +150,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA GCS_PATH_FORMAT, bucket, String.format(FILENAME_FORMAT, tld, exportTime))) .collect(toImmutableList()); return ImmutableMap.of( - "jobPath", createJobPath(jobId), + "mapreduceConsoleLink", mapreduceConsoleLink, "filenames", filenames); } diff --git a/java/google/registry/tools/server/KillAllCommitLogsAction.java b/java/google/registry/tools/server/KillAllCommitLogsAction.java index fc3aa4e40..6aa78da4f 100644 --- a/java/google/registry/tools/server/KillAllCommitLogsAction.java +++ b/java/google/registry/tools/server/KillAllCommitLogsAction.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.request.Action.Method.POST; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.mapreduce.Input; import com.google.appengine.tools.mapreduce.Mapper; @@ -72,13 +71,12 @@ public class KillAllCommitLogsAction implements Runnable { CommitLogBucket.getAllBucketKeys().stream()) .collect(toImmutableList()), 1)); - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Delete all commit logs") .setModuleName("tools") .runMapreduce( - new KillAllCommitLogsMapper(), - new KillAllEntitiesReducer(), - ImmutableList.of(input)))); + new KillAllCommitLogsMapper(), new KillAllEntitiesReducer(), ImmutableList.of(input)) + .sendLinkToMapreduceConsole(response); } /** diff --git a/java/google/registry/tools/server/KillAllEppResourcesAction.java b/java/google/registry/tools/server/KillAllEppResourcesAction.java index 2879e7708..c71e8355a 100644 --- a/java/google/registry/tools/server/KillAllEppResourcesAction.java +++ b/java/google/registry/tools/server/KillAllEppResourcesAction.java @@ -17,7 +17,6 @@ package google.registry.tools.server; import static com.google.common.base.Preconditions.checkArgument; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.request.Action.Method.POST; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.mapreduce.Mapper; import com.google.common.collect.ImmutableList; @@ -58,13 +57,14 @@ public class KillAllEppResourcesAction implements Runnable { RegistryEnvironment.get() == RegistryEnvironment.CRASH || RegistryEnvironment.get() == RegistryEnvironment.UNITTEST, "DO NOT RUN ANYWHERE ELSE EXCEPT CRASH OR TESTS."); - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Delete all EppResources, children, and indices") .setModuleName("tools") .runMapreduce( new KillAllEppResourcesMapper(), new KillAllEntitiesReducer(), - ImmutableList.of(EppResourceInputs.createIndexInput())))); + ImmutableList.of(EppResourceInputs.createIndexInput())) + .sendLinkToMapreduceConsole(response); } static class KillAllEppResourcesMapper extends Mapper, Key> { diff --git a/java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java b/java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java index d9479dfe0..63cfa5b00 100644 --- a/java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java +++ b/java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java @@ -18,7 +18,6 @@ import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInp import static google.registry.model.EppResourceUtils.isActive; import static google.registry.model.registry.Registries.assertTldsExist; import static google.registry.request.RequestParameters.PARAM_TLDS; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.mapreduce.Mapper; import com.google.common.annotations.VisibleForTesting; @@ -64,15 +63,14 @@ public class RefreshDnsForAllDomainsAction implements Runnable { @Override public void run() { assertTldsExist(tlds); - response.sendJavaScriptRedirect( - createJobPath( - mrRunner - .setJobName("Refresh DNS for all domains") - .setModuleName("tools") - .setDefaultMapShards(10) - .runMapOnly( - new RefreshDnsForAllDomainsActionMapper(tlds), - ImmutableList.of(createEntityInput(DomainResource.class))))); + mrRunner + .setJobName("Refresh DNS for all domains") + .setModuleName("tools") + .setDefaultMapShards(10) + .runMapOnly( + new RefreshDnsForAllDomainsActionMapper(tlds), + ImmutableList.of(createEntityInput(DomainResource.class))) + .sendLinkToMapreduceConsole(response); } /** Mapper to refresh DNS for all active domain resources. */ diff --git a/java/google/registry/tools/server/ResaveAllHistoryEntriesAction.java b/java/google/registry/tools/server/ResaveAllHistoryEntriesAction.java index b0d22b560..d86c86be5 100644 --- a/java/google/registry/tools/server/ResaveAllHistoryEntriesAction.java +++ b/java/google/registry/tools/server/ResaveAllHistoryEntriesAction.java @@ -15,7 +15,6 @@ package google.registry.tools.server; import static google.registry.model.ofy.ObjectifyService.ofy; -import static google.registry.util.PipelineUtils.createJobPath; import com.google.appengine.tools.mapreduce.Mapper; import com.google.common.collect.ImmutableList; @@ -51,14 +50,15 @@ public class ResaveAllHistoryEntriesAction implements Runnable { @SuppressWarnings("unchecked") @Override public void run() { - response.sendJavaScriptRedirect(createJobPath(mrRunner + mrRunner .setJobName("Re-save all HistoryEntry entities") .setModuleName("tools") .runMapOnly( new ResaveAllHistoryEntriesActionMapper(), - ImmutableList.of(EppResourceInputs.createChildEntityInput( - ImmutableSet.of(EppResource.class), - ImmutableSet.of(HistoryEntry.class)))))); + ImmutableList.of( + EppResourceInputs.createChildEntityInput( + ImmutableSet.of(EppResource.class), ImmutableSet.of(HistoryEntry.class)))) + .sendLinkToMapreduceConsole(response); } /** Mapper to re-save all HistoryEntry entities. */ diff --git a/java/google/registry/util/PipelineUtils.java b/java/google/registry/util/PipelineUtils.java deleted file mode 100644 index 63e4ef3cd..000000000 --- a/java/google/registry/util/PipelineUtils.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.util; - -/** Utilities for working with pipeline jobs. */ -public class PipelineUtils { - - private static final String URL_PREFIX = "/_ah/pipeline/status.html?root="; - - private PipelineUtils() {} - - public static String createJobPath(String jobId) { - return URL_PREFIX + jobId; - } -} diff --git a/javatests/google/registry/export/ExportDomainListsActionTest.java b/javatests/google/registry/export/ExportDomainListsActionTest.java index 4bd09e3aa..36a528920 100644 --- a/javatests/google/registry/export/ExportDomainListsActionTest.java +++ b/javatests/google/registry/export/ExportDomainListsActionTest.java @@ -55,6 +55,7 @@ public class ExportDomainListsActionTest extends MapreduceTestCase bytesExportedToDrive = ArgumentCaptor.forClass(byte[].class); + private final FakeResponse response = new FakeResponse(); @Before public void init() { @@ -67,7 +68,7 @@ public class ExportDomainListsActionTest extends MapreduceTestCase inject.setStaticField(Ofy.class, "clock", clock); createTld("test"); response = new FakeResponse(); - mrRunner = new MapreduceRunner(Optional.empty(), Optional.empty()); + mrRunner = makeDefaultRunner(); action = new RdeHostLinkAction(mrRunner, response, IMPORT_BUCKET_NAME, IMPORT_FILE_NAME, mapShards); } diff --git a/javatests/google/registry/request/ResponseImplTest.java b/javatests/google/registry/request/ResponseImplTest.java index c6d32429b..355b52066 100644 --- a/javatests/google/registry/request/ResponseImplTest.java +++ b/javatests/google/registry/request/ResponseImplTest.java @@ -55,13 +55,4 @@ public class ResponseImplTest { new ResponseImpl(rsp).setPayload("hello world"); assertThat(httpOutput.toString()).isEqualTo("hello world"); } - - @Test - public void testSendJavaScriptRedirect_producesHtmlScript() throws Exception { - StringWriter httpOutput = new StringWriter(); - when(rsp.getWriter()).thenReturn(new PrintWriter(httpOutput)); - new ResponseImpl(rsp).sendJavaScriptRedirect("/hello"); - assertThat(httpOutput.toString()).isEqualTo( - "/hello"); - } } diff --git a/javatests/google/registry/testing/FakeResponse.java b/javatests/google/registry/testing/FakeResponse.java index 19d4e5d0f..921cb8897 100644 --- a/javatests/google/registry/testing/FakeResponse.java +++ b/javatests/google/registry/testing/FakeResponse.java @@ -80,16 +80,11 @@ public final class FakeResponse implements Response { headers.put(checkNotNull(header), checkNotNull(timestamp)); } - @Override - public void sendJavaScriptRedirect(String redirectUrl) { - checkResponsePerformedOnce(); - this.status = 200; - this.payload = "Javascript redirect to " + redirectUrl; - } - private void checkResponsePerformedOnce() { - checkState(!wasMutuallyExclusiveResponseSet, - "Two responses were sent. Here's the previous call:\n%s", lastResponseStackTrace); + checkState( + !wasMutuallyExclusiveResponseSet, + "Two responses were sent. Here's the previous call:\n%s", + lastResponseStackTrace); wasMutuallyExclusiveResponseSet = true; lastResponseStackTrace = getStackTrace(); } diff --git a/javatests/google/registry/testing/mapreduce/BUILD b/javatests/google/registry/testing/mapreduce/BUILD index db26917ba..ad1a8055e 100644 --- a/javatests/google/registry/testing/mapreduce/BUILD +++ b/javatests/google/registry/testing/mapreduce/BUILD @@ -14,6 +14,7 @@ java_library( "//java/google/registry/config", "//java/google/registry/mapreduce", "//java/google/registry/model", + "//java/google/registry/util", "//javatests/google/registry/testing", "@com_google_appengine_api_1_0_sdk", "@com_google_appengine_api_stubs", diff --git a/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java b/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java index d853d38ab..7e0452895 100644 --- a/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java +++ b/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java @@ -14,7 +14,6 @@ package google.registry.testing.mapreduce; -import static com.google.common.truth.Truth.assertThat; import static google.registry.config.RegistryConfig.getEppResourceIndexBucketCount; import static google.registry.model.ofy.ObjectifyService.ofy; import static org.mockito.Mockito.mock; @@ -24,11 +23,9 @@ import com.google.appengine.api.blobstore.dev.LocalBlobstoreService; import com.google.appengine.api.taskqueue.dev.LocalTaskQueue; import com.google.appengine.api.taskqueue.dev.QueueStateInfo; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.HeaderWrapper; -import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; import com.google.appengine.tools.development.ApiProxyLocal; import com.google.appengine.tools.development.testing.LocalTaskQueueTestConfig; import com.google.appengine.tools.mapreduce.MapReduceServlet; -import com.google.appengine.tools.mapreduce.impl.shardedjob.ShardedJobHandler; import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; import com.google.appengine.tools.pipeline.impl.servlets.TaskHandler; import com.google.apphosting.api.ApiProxy; @@ -37,7 +34,9 @@ import com.google.common.flogger.FluentLogger; import google.registry.mapreduce.MapreduceRunner; import google.registry.testing.AppEngineRule; import google.registry.testing.FakeClock; +import google.registry.testing.MockitoJUnitRule; import google.registry.testing.ShardableTestCase; +import google.registry.util.AppEngineServiceUtils; import java.io.ByteArrayInputStream; import java.io.ObjectInputStream; import java.io.UnsupportedEncodingException; @@ -52,13 +51,16 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.junit.Before; import org.junit.Rule; +import org.mockito.Mock; /** - * Base test class for mapreduces. Adapted from EndToEndTestCase with some modifications that - * allow it to work with the Nomulus project, most notably inside knowledge of our - * routing paths and our Datastore/Task Queue configurations. + * Base test class for mapreduces. * - *

See https://github.com/GoogleCloudPlatform/appengine-mapreduce/blob/master/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java + *

Adapted from EndToEndTestCase with some modifications that allow it to work with Nomulus, most + * notably inside knowledge of our routing paths and our Datastore/Task Queue configurations. + * + *

See + * https://github.com/GoogleCloudPlatform/appengine-mapreduce/blob/master/java/src/test/java/com/google/appengine/tools/mapreduce/EndToEndTestCase.java * * @param The type of the Action class that implements the mapreduce. */ @@ -79,16 +81,23 @@ public abstract class MapreduceTestCase extends ShardableTestCase { .withTaskQueue() .build(); + @Rule public final MockitoJUnitRule mocks = MockitoJUnitRule.create(); + + @Mock + AppEngineServiceUtils appEngineServiceUtils; + @Before public void setUp() { taskQueue = LocalTaskQueueTestConfig.getLocalTaskQueue(); ApiProxyLocal proxy = (ApiProxyLocal) ApiProxy.getDelegate(); // Creating files is not allowed in some test execution environments, so don't. proxy.setProperty(LocalBlobstoreService.NO_STORAGE_PROPERTY, "true"); + when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.tld"); } protected MapreduceRunner makeDefaultRunner() { - return new MapreduceRunner(Optional.of(getEppResourceIndexBucketCount()), Optional.of(1)); + return new MapreduceRunner( + Optional.of(getEppResourceIndexBucketCount()), Optional.of(1), appEngineServiceUtils); } protected List getTasks(String queueName) { @@ -210,14 +219,6 @@ public abstract class MapreduceTestCase extends ShardableTestCase { } } - protected TaskStateInfo grabNextTaskFromQueue(String queueName) { - List taskInfo = getTasks(queueName); - assertThat(taskInfo).isNotEmpty(); - TaskStateInfo taskStateInfo = taskInfo.get(0); - taskQueue.deleteTask(queueName, taskStateInfo.getTaskName()); - return taskStateInfo; - } - // Sadly there's no way to parse query string with JDK. This is a good enough approximation. private static Map decodeParameters(String requestBody) throws UnsupportedEncodingException { @@ -236,8 +237,4 @@ public abstract class MapreduceTestCase extends ShardableTestCase { return result; } - - protected String getTaskId(TaskStateInfo taskStateInfo) throws UnsupportedEncodingException { - return decodeParameters(taskStateInfo.getBody()).get(ShardedJobHandler.TASK_ID_PARAM); - } } diff --git a/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java b/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java index d3453efd6..6c60151c6 100644 --- a/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java +++ b/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java @@ -16,7 +16,7 @@ package google.registry.tools.server; import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService; import static com.google.common.truth.Truth.assertThat; -import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.DatastoreHelper.createTlds; import static google.registry.testing.DatastoreHelper.newDomainResource; import static google.registry.testing.DatastoreHelper.newHostResource; import static google.registry.testing.DatastoreHelper.persistActiveContact; @@ -58,9 +58,7 @@ public class GenerateZoneFilesActionTest extends MapreduceTestCase ips = ImmutableSet.of(InetAddress.getByName("127.0.0.1"), InetAddress.getByName("::1")); @@ -126,12 +124,15 @@ public class GenerateZoneFilesActionTest extends MapreduceTestCase response = action.handleJsonRequest(ImmutableMap.of( - "tlds", ImmutableList.of("tld"), - "exportTime", now)); - assertThat(response).containsEntry( - "filenames", - ImmutableList.of("gs://zonefiles-bucket/tld-" + now + ".zone")); + Map response = + action.handleJsonRequest( + ImmutableMap.of("tlds", ImmutableList.of("tld"), "exportTime", now)); + assertThat(response) + .containsEntry("filenames", ImmutableList.of("gs://zonefiles-bucket/tld-" + now + ".zone")); + assertThat(response).containsKey("mapreduceConsoleLink"); + assertThat(response.get("mapreduceConsoleLink").toString()) + .startsWith( + "Mapreduce console: https://backend.hostname.tld/_ah/pipeline/status.html?root="); executeTasksUntilEmpty("mapreduce");