zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-19 02:39:34 +02:00
Code Issues Projects Releases Packages Wiki Activity

Send a plaintext link to the mapreduce console in fluent style

Browse source 
The link was previously being sent using a JS redirect, which doesn't work
because the endpoints that trigger mapreduces can only be hit from the command
line (because they require auth). This commit switches the link to be in
plaintext and renders the full URL instead of just the path, so that clicking it
directly from the terminal works.

This also improves how these links are sent from callsites by using a fluent
style.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=228764606
This commit is contained in:
mcilwain 2019-01-10 13:20:08 -08:00 • committed by Ben McIlwain
parent 072576ec9d
commit 765e63e7e9
31 changed files with 202 additions and 239 deletions
Download patch file Download diff file Expand all files Collapse all files

24
java/google/registry/backup/DeleteOldCommitLogsAction.java
View file

@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.lang.Boolean.FALSE;
import static java.lang.Boolean.TRUE;
@ -88,17 +87,18 @@ public final class DeleteOldCommitLogsAction implements Runnable {
"Processing asynchronous deletion of unreferenced CommitLogManifests older than %s",
deletionThreshold);
response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Delete old commit logs")
.setModuleName("backend")
.setDefaultMapShards(NUM_MAP_SHARDS)
.setDefaultReduceShards(NUM_REDUCE_SHARDS)
.runMapreduce(
new DeleteOldCommitLogsMapper(deletionThreshold),
new DeleteOldCommitLogsReducer(deletionThreshold, isDryRun),
ImmutableList.of(
new CommitLogManifestInput(deletionThreshold),
EppResourceInputs.createKeyInput(EppResource.class)))));
mrRunner
.setJobName("Delete old commit logs")
.setModuleName("backend")
.setDefaultMapShards(NUM_MAP_SHARDS)
.setDefaultReduceShards(NUM_REDUCE_SHARDS)
.runMapreduce(
new DeleteOldCommitLogsMapper(deletionThreshold),
new DeleteOldCommitLogsReducer(deletionThreshold, isDryRun),
ImmutableList.of(
new CommitLogManifestInput(deletionThreshold),
EppResourceInputs.createKeyInput(EppResource.class)))
.sendLinkToMapreduceConsole(response);
}
/**

27
java/google/registry/batch/DeleteContactsAndHostsAction.java
View file

@ -40,7 +40,6 @@ import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE_F
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE;
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE_FAILURE;
import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.math.RoundingMode.CEILING;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.SECONDS;
@ -215,20 +214,18 @@ public class DeleteContactsAndHostsAction implements Runnable {
try {
int numReducers =
Math.min(MAX_REDUCE_SHARDS, divide(deletionRequests.size(), DELETES_PER_SHARD, CEILING));
response.sendJavaScriptRedirect(
createJobPath(
mrRunner
.setJobName("Check for EPP resource references and then delete")
.setModuleName("backend")
.setDefaultReduceShards(numReducers)
.runMapreduce(
new DeleteContactsAndHostsMapper(deletionRequests),
new DeleteEppResourceReducer(),
ImmutableList.of(
// Add an extra shard that maps over a null domain. See the mapper code
// for why.
new NullInput<>(), EppResourceInputs.createEntityInput(DomainBase.class)),
new UnlockerOutput<Void>(lock.get()))));
mrRunner
.setJobName("Check for EPP resource references and then delete")
.setModuleName("backend")
.setDefaultReduceShards(numReducers)
.runMapreduce(
new DeleteContactsAndHostsMapper(deletionRequests),
new DeleteEppResourceReducer(),
ImmutableList.of(
// Add an extra shard that maps over a null domain. See the mapper code for why.
new NullInput<>(), EppResourceInputs.createEntityInput(DomainBase.class)),
new UnlockerOutput<Void>(lock.get()))
.sendLinkToMapreduceConsole(response);
} catch (Throwable t) {
logRespondAndUnlock(SEVERE, "Error starting mapreduce to delete contacts/hosts.", lock);
}

19
java/google/registry/batch/DeleteLoadTestDataAction.java
View file

@ -37,7 +37,6 @@ import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.PipelineUtils;
import java.util.List;
import javax.inject.Inject;
@ -83,16 +82,14 @@ public class DeleteLoadTestDataAction implements Runnable {
checkState(
registryEnvironment != PRODUCTION, "This mapreduce is not safe to run on PRODUCTION.");
response.sendJavaScriptRedirect(
PipelineUtils.createJobPath(
mrRunner
.setJobName("Delete load test data")
.setModuleName("backend")
.runMapOnly(
new DeleteLoadTestDataMapper(isDryRun),
ImmutableList.of(
createEntityInput(ContactResource.class),
createEntityInput(HostResource.class)))));
mrRunner
.setJobName("Delete load test data")
.setModuleName("backend")
.runMapOnly(
new DeleteLoadTestDataMapper(isDryRun),
ImmutableList.of(
createEntityInput(ContactResource.class), createEntityInput(HostResource.class)))
.sendLinkToMapreduceConsole(response);
}
/** Provides the map method that runs for each existing contact and host entity. */

6
java/google/registry/batch/DeleteProberDataAction.java
View file

@ -52,7 +52,6 @@ import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.PipelineUtils;
import java.util.List;
import javax.inject.Inject;
import org.joda.time.DateTime;
@ -87,12 +86,13 @@ public class DeleteProberDataAction implements Runnable {
checkState(
!Strings.isNullOrEmpty(registryAdminClientId),
"Registry admin client ID must be configured for prober data deletion to work");
response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner
mrRunner
.setJobName("Delete prober data")
.setModuleName("backend")
.runMapOnly(
new DeleteProberDataMapper(getProberRoidSuffixes(), isDryRun, registryAdminClientId),
ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))));
ImmutableList.of(EppResourceInputs.createKeyInput(DomainBase.class)))
.sendLinkToMapreduceConsole(response);
}
private ImmutableSet<String> getProberRoidSuffixes() {

6
java/google/registry/batch/ExpandRecurringBillingEventsAction.java
View file

@ -28,7 +28,6 @@ import static google.registry.util.CollectionUtils.union;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static google.registry.util.DateTimeUtils.earliestOf;
import static google.registry.util.DomainNameUtils.getTldFromDomainName;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.Reducer;
@ -102,7 +101,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
logger.atInfo().log(
"Running Recurring billing event expansion for billing time range [%s, %s).",
cursorTime, executeTime);
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Expand Recurring billing events into synthetic OneTime events.")
.setModuleName("backend")
.runMapreduce(
@ -112,7 +111,8 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
ImmutableList.of(
new NullInput<>(),
createChildEntityInput(
ImmutableSet.of(DomainResource.class), ImmutableSet.of(Recurring.class))))));
ImmutableSet.of(DomainResource.class), ImmutableSet.of(Recurring.class))))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to expand {@link Recurring} billing events into synthetic {@link OneTime} events. */

23
java/google/registry/batch/RefreshDnsOnHostRenameAction.java
View file

@ -27,7 +27,6 @@ import static google.registry.model.EppResourceUtils.isActive;
import static google.registry.model.EppResourceUtils.isDeleted;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.DateTimeUtils.latestOf;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.logging.Level.INFO;
@ -159,18 +158,16 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
private void runMapreduce(ImmutableList<DnsRefreshRequest> refreshRequests, Optional<Lock> lock) {
try {
response.sendJavaScriptRedirect(
createJobPath(
mrRunner
.setJobName("Enqueue DNS refreshes for domains referencing renamed hosts")
.setModuleName("backend")
.setDefaultReduceShards(1)
.runMapreduce(
new RefreshDnsOnHostRenameMapper(refreshRequests, retrier),
new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier),
// Add an extra NullInput so that the reducer always fires exactly once.
ImmutableList.of(
new NullInput<>(), createEntityInput(DomainResource.class)))));
mrRunner
.setJobName("Enqueue DNS refreshes for domains referencing renamed hosts")
.setModuleName("backend")
.setDefaultReduceShards(1)
.runMapreduce(
new RefreshDnsOnHostRenameMapper(refreshRequests, retrier),
new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier),
// Add an extra NullInput so that the reducer always fires exactly once.
ImmutableList.of(new NullInput<>(), createEntityInput(DomainResource.class)))
.sendLinkToMapreduceConsole(response);
} catch (Throwable t) {
logRespondAndUnlock(
SEVERE, "Error starting mapreduce to refresh DNS for renamed hosts.", lock);

6
java/google/registry/batch/ResaveAllEppResourcesAction.java
View file

@ -15,7 +15,6 @@
package google.registry.batch;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
@ -52,12 +51,13 @@ public class ResaveAllEppResourcesAction implements Runnable {
@Override
public void run() {
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Re-save all EPP resources")
.setModuleName("backend")
.runMapOnly(
new ResaveAllEppResourcesActionMapper(),
ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))));
ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to re-save all EPP resources. */

6
java/google/registry/export/ExportDomainListsAction.java
View file

@ -20,7 +20,6 @@ import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInp
import static google.registry.model.EppResourceUtils.isActive;
import static google.registry.model.registry.Registries.getTldsOfType;
import static google.registry.request.Action.Method.POST;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.joda.time.DateTimeZone.UTC;
@ -78,14 +77,15 @@ public class ExportDomainListsAction implements Runnable {
public void run() {
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
logger.atInfo().log("Exporting domain lists for tlds %s", realTlds);
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Export domain lists")
.setModuleName("backend")
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
.runMapreduce(
new ExportDomainListsMapper(DateTime.now(UTC), realTlds),
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
ImmutableList.of(createEntityInput(DomainResource.class)))));
ImmutableList.of(createEntityInput(DomainResource.class)))
.sendLinkToMapreduceConsole(response);
}
static class ExportDomainListsMapper extends Mapper<DomainResource, String, String> {

80
java/google/registry/mapreduce/MapreduceRunner.java
View file

@ -35,7 +35,8 @@ import com.google.appengine.tools.pipeline.JobSetting;
import com.google.common.flogger.FluentLogger;
import google.registry.mapreduce.inputs.ConcatenatingInput;
import google.registry.request.Parameter;
import google.registry.util.PipelineUtils;
import google.registry.request.Response;
import google.registry.util.AppEngineServiceUtils;
import java.io.Serializable;
import java.util.Optional;
import javax.inject.Inject;
@ -58,8 +59,12 @@ public class MapreduceRunner {
private static final String BASE_URL = "/_dr/mapreduce/";
private static final String QUEUE_NAME = "mapreduce";
private static final String MAPREDUCE_CONSOLE_LINK_FORMAT =
"Mapreduce console: https://%s/_ah/pipeline/status.html?root=%s";
private final Optional<Integer> httpParamMapShards;
private final Optional<Integer> httpParamReduceShards;
private final AppEngineServiceUtils appEngineServiceUtils;
// Default to 3 minutes since many slices will contain Datastore queries that time out at 4:30.
private Duration sliceDuration = Duration.standardMinutes(3);
@ -87,9 +92,11 @@ public class MapreduceRunner {
@Inject
public MapreduceRunner(
@Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards,
@Parameter(PARAM_REDUCE_SHARDS) Optional<Integer> reduceShards) {
@Parameter(PARAM_REDUCE_SHARDS) Optional<Integer> reduceShards,
AppEngineServiceUtils appEngineServiceUtils) {
this.httpParamMapShards = mapShards;
this.httpParamReduceShards = reduceShards;
this.appEngineServiceUtils = appEngineServiceUtils;
}
/** Set the max time to run a slice before serializing; defaults to 3 minutes. */
@ -160,15 +167,13 @@ public class MapreduceRunner {
* all work will be accomplished via side effects during the map phase.
*
* @see #createMapOnlyJob for creating and running a map-only mapreduce as part of a pipeline
*
* @param mapper instance of a mapper class
* @param inputs input sources for the mapper
* @param <I> mapper input type
* @return the job id
*/
public <I> String runMapOnly(
Mapper<I, Void, Void> mapper,
Iterable<? extends Input<? extends I>> inputs) {
public <I> MapreduceRunnerResult runMapOnly(
Mapper<I, Void, Void> mapper, Iterable<? extends Input<? extends I>> inputs) {
return runAsPipeline(createMapOnlyJob(mapper, new NoOutput<Void, Void>(), inputs));
}
@ -220,7 +225,6 @@ public class MapreduceRunner {
* all work will be accomplished via side effects during the map or reduce phases.
*
* @see #createMapreduceJob for creating and running a mapreduce as part of a pipeline
*
* @param mapper instance of a mapper class
* @param reducer instance of a reducer class
* @param inputs input sources for the mapper
@ -229,10 +233,11 @@ public class MapreduceRunner {
* @param <V> emitted value type
* @return the job id
*/
public final <I, K extends Serializable, V extends Serializable> String runMapreduce(
Mapper<I, K, V> mapper,
Reducer<K, V, Void> reducer,
Iterable<? extends Input<? extends I>> inputs) {
public final <I, K extends Serializable, V extends Serializable>
MapreduceRunnerResult runMapreduce(
Mapper<I, K, V> mapper,
Reducer<K, V, Void> reducer,
Iterable<? extends Input<? extends I>> inputs) {
return runMapreduce(mapper, reducer, inputs, new NoOutput<Void, Void>());
}
@ -240,7 +245,6 @@ public class MapreduceRunner {
* Kick off a mapreduce job with specified Output handler.
*
* @see #createMapreduceJob for creating and running a mapreduce as part of a pipeline
*
* @param mapper instance of a mapper class
* @param reducer instance of a reducer class
* @param inputs input sources for the mapper
@ -251,11 +255,12 @@ public class MapreduceRunner {
* @param <R> return value of output
* @return the job id
*/
public final <I, K extends Serializable, V extends Serializable, O, R> String runMapreduce(
Mapper<I, K, V> mapper,
Reducer<K, V, O> reducer,
Iterable<? extends Input<? extends I>> inputs,
Output<O, R> output) {
public final <I, K extends Serializable, V extends Serializable, O, R>
MapreduceRunnerResult runMapreduce(
Mapper<I, K, V> mapper,
Reducer<K, V, O> reducer,
Iterable<? extends Input<? extends I>> inputs,
Output<O, R> output) {
return runAsPipeline(createMapreduceJob(mapper, reducer, inputs, output));
}
@ -266,14 +271,41 @@ public class MapreduceRunner {
checkArgumentNotNull(mapper, "mapper");
}
private String runAsPipeline(Job0<?> job) {
String jobId = newPipelineService().startNewPipeline(
job,
new JobSetting.OnModule(moduleName),
new JobSetting.OnQueue(QUEUE_NAME));
private MapreduceRunnerResult runAsPipeline(Job0<?> job) {
String jobId =
newPipelineService()
.startNewPipeline(
job, new JobSetting.OnModule(moduleName), new JobSetting.OnQueue(QUEUE_NAME));
logger.atInfo().log(
"Started '%s' %s job: %s",
jobName, job instanceof MapJob ? "map" : "mapreduce", PipelineUtils.createJobPath(jobId));
return jobId;
jobName, job instanceof MapJob ? "map" : "mapreduce", renderMapreduceConsoleLink(jobId));
return new MapreduceRunnerResult(jobId);
}
private String renderMapreduceConsoleLink(String jobId) {
return String.format(
MAPREDUCE_CONSOLE_LINK_FORMAT, appEngineServiceUtils.getServiceHostname("backend"), jobId);
}
/**
* Class representing the result of kicking off a mapreduce.
*
* <p>This is used to send a link to the mapreduce console.
*/
public class MapreduceRunnerResult {
private final String jobId;
private MapreduceRunnerResult(String jobId) {
this.jobId = jobId;
}
public void sendLinkToMapreduceConsole(Response response) {
response.setPayload(getLinkToMapreduceConsole());
}
public String getLinkToMapreduceConsole() {
return renderMapreduceConsoleLink(jobId);
}
}
}

7
java/google/registry/rde/RdeStagingAction.java
View file

@ -16,7 +16,6 @@ package google.registry.rde;
import static google.registry.request.Action.Method.GET;
import static google.registry.request.Action.Method.POST;
import static google.registry.util.PipelineUtils.createJobPath;
import static google.registry.xml.ValidationMode.LENIENT;
import static google.registry.xml.ValidationMode.STRICT;
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
@ -228,7 +227,7 @@ public final class RdeStagingAction implements Runnable {
}
RdeStagingMapper mapper = new RdeStagingMapper(lenient ? LENIENT : STRICT, pendings);
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Stage escrow deposits for all TLDs")
.setModuleName("backend")
.setDefaultReduceShards(pendings.size())
@ -237,8 +236,8 @@ public final class RdeStagingAction implements Runnable {
reducer,
ImmutableList.of(
// Add an extra shard that maps over a null resource. See the mapper code for why.
new NullInput<>(),
EppResourceInputs.createEntityInput(EppResource.class)))));
new NullInput<>(), EppResourceInputs.createEntityInput(EppResource.class)))
.sendLinkToMapreduceConsole(response);
}
private ImmutableSetMultimap<String, PendingDeposit> getStandardPendingDeposits() {

8
java/google/registry/rde/imports/RdeContactImportAction.java
View file

@ -16,7 +16,6 @@ package google.registry.rde.imports;
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
@ -77,12 +76,11 @@ public class RdeContactImportAction implements Runnable {
@Override
public void run() {
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Import contacts from escrow file")
.setModuleName("backend")
.runMapOnly(
createMapper(),
ImmutableList.of(createInput()))));
.runMapOnly(createMapper(), ImmutableList.of(createInput()))
.sendLinkToMapreduceConsole(response);
}
/**

8
java/google/registry/rde/imports/RdeDomainImportAction.java
View file

@ -25,7 +25,6 @@ import static google.registry.rde.imports.RdeImportUtils.createAutoRenewBillingE
import static google.registry.rde.imports.RdeImportUtils.createAutoRenewPollMessageForDomainImport;
import static google.registry.rde.imports.RdeImportUtils.createHistoryEntryForDomainImport;
import static google.registry.rde.imports.RdeImportsModule.PATH;
import static google.registry.util.PipelineUtils.createJobPath;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.google.appengine.tools.cloudstorage.GcsService;
@ -107,12 +106,11 @@ public class RdeDomainImportAction implements Runnable {
logger.atInfo().log(
"Launching domains import mapreduce: bucket=%s, filename=%s",
this.importBucketName, this.importFileName);
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Import domains from escrow file")
.setModuleName("backend")
.runMapOnly(
createMapper(),
ImmutableList.of(createInput()))));
.runMapOnly(createMapper(), ImmutableList.of(createInput()))
.sendLinkToMapreduceConsole(response);
}
/**

6
java/google/registry/rde/imports/RdeHostImportAction.java
View file

@ -16,7 +16,6 @@ package google.registry.rde.imports;
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
@ -77,12 +76,13 @@ public class RdeHostImportAction implements Runnable {
@Override
public void run() {
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Import hosts from escrow file")
.setModuleName("backend")
.runMapOnly(
new RdeHostImportMapper(importBucketName),
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))));
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to import hosts from an escrow file. */

6
java/google/registry/rde/imports/RdeHostLinkAction.java
View file

@ -19,7 +19,6 @@ import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.registry.Registries.findTldForName;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.util.stream.Collectors.joining;
import com.google.appengine.tools.mapreduce.Mapper;
@ -86,12 +85,13 @@ public class RdeHostLinkAction implements Runnable {
@Override
public void run() {
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Link hosts from escrow file")
.setModuleName("backend")
.runMapOnly(
new RdeHostPostImportMapper(),
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))));
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to link hosts from an escrow file to their superordinate domains. */

8
java/google/registry/request/Response.java
View file

@ -51,12 +51,4 @@ public interface Response {
* @see HttpServletResponse#setDateHeader(String, long)
*/
void setDateHeader(String header, DateTime timestamp);
/**
* Sends a JavaScript redirect HTTP response.
*
* GAE handles a HTTP 302 status as an error, so using this is helpful for responses that might
* sometimes be consumed by GAE code, since it performs a redirect while also returning HTTP 200.
*/
void sendJavaScriptRedirect(String redirectUrl);
}

9
java/google/registry/request/ResponseImpl.java
View file

@ -23,10 +23,6 @@ import org.joda.time.DateTime;
/** HTTP response object. */
public final class ResponseImpl implements Response {
/** Code for a JavaScript redirect. */
private static final String REDIRECT_PAYLOAD_FORMAT =
"<script>window.location.replace(\"%1$s\");</script><a href=\"%1$s\">%1$s</a>";
private final HttpServletResponse rsp;
@Inject
@ -62,9 +58,4 @@ public final class ResponseImpl implements Response {
public void setDateHeader(String header, DateTime timestamp) {
rsp.setDateHeader(header, timestamp.getMillis());
}
@Override
public void sendJavaScriptRedirect(String redirectUrl) {
setPayload(String.format(REDIRECT_PAYLOAD_FORMAT, redirectUrl));
}
}

2
java/google/registry/tools/GenerateZoneFilesCommand.java
View file

@ -59,7 +59,7 @@ final class GenerateZoneFilesCommand implements CommandWithConnection, CommandWi
"tlds", mainParameters,
"exportTime", exportDate.toString());
Map<String, Object> response = connection.sendJson(GenerateZoneFilesAction.PATH, params);
System.out.printf("Job started at %s %s\n", connection.getServer(), response.get("jobPath"));
System.out.println(response.get("mapreduceConsoleLink"));
System.out.println("Output files:");
@SuppressWarnings("unchecked")
List<String> filenames = (List<String>) response.get("filenames");

25
java/google/registry/tools/server/GenerateZoneFilesAction.java
View file

@ -22,7 +22,6 @@ import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInp
import static google.registry.model.EppResourceUtils.loadAtPointInTime;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.request.Action.Method.POST;
import static google.registry.util.PipelineUtils.createJobPath;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.joda.time.DateTimeZone.UTC;
@ -132,17 +131,17 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
if (!exportTime.equals(exportTime.toDateTime(UTC).withTimeAtStartOfDay())) {
throw new BadRequestException("Invalid export time: must be midnight UTC");
}
String jobId = mrRunner
.setJobName("Generate bind file stanzas")
.setModuleName("tools")
.setDefaultReduceShards(tlds.size())
.runMapreduce(
new GenerateBindFileMapper(
tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl),
new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize),
ImmutableList.of(
new NullInput<>(),
createEntityInput(DomainResource.class)));
String mapreduceConsoleLink =
mrRunner
.setJobName("Generate bind file stanzas")
.setModuleName("tools")
.setDefaultReduceShards(tlds.size())
.runMapreduce(
new GenerateBindFileMapper(
tlds, exportTime, dnsDefaultATtl, dnsDefaultNsTtl, dnsDefaultDsTtl),
new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize),
ImmutableList.of(new NullInput<>(), createEntityInput(DomainResource.class)))
.getLinkToMapreduceConsole();
ImmutableList<String> filenames =
tlds.stream()
.map(
@ -151,7 +150,7 @@ public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonA
GCS_PATH_FORMAT, bucket, String.format(FILENAME_FORMAT, tld, exportTime)))
.collect(toImmutableList());
return ImmutableMap.of(
"jobPath", createJobPath(jobId),
"mapreduceConsoleLink", mapreduceConsoleLink,
"filenames", filenames);
}

8
java/google/registry/tools/server/KillAllCommitLogsAction.java
View file

@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.request.Action.Method.POST;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Input;
import com.google.appengine.tools.mapreduce.Mapper;
@ -72,13 +71,12 @@ public class KillAllCommitLogsAction implements Runnable {
CommitLogBucket.getAllBucketKeys().stream())
.collect(toImmutableList()),
1));
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Delete all commit logs")
.setModuleName("tools")
.runMapreduce(
new KillAllCommitLogsMapper(),
new KillAllEntitiesReducer(),
ImmutableList.of(input))));
new KillAllCommitLogsMapper(), new KillAllEntitiesReducer(), ImmutableList.of(input))
.sendLinkToMapreduceConsole(response);
}
/**

6
java/google/registry/tools/server/KillAllEppResourcesAction.java
View file

@ -17,7 +17,6 @@ package google.registry.tools.server;
import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.request.Action.Method.POST;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
@ -58,13 +57,14 @@ public class KillAllEppResourcesAction implements Runnable {
RegistryEnvironment.get() == RegistryEnvironment.CRASH
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
"DO NOT RUN ANYWHERE ELSE EXCEPT CRASH OR TESTS.");
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Delete all EppResources, children, and indices")
.setModuleName("tools")
.runMapreduce(
new KillAllEppResourcesMapper(),
new KillAllEntitiesReducer(),
ImmutableList.of(EppResourceInputs.createIndexInput()))));
ImmutableList.of(EppResourceInputs.createIndexInput()))
.sendLinkToMapreduceConsole(response);
}
static class KillAllEppResourcesMapper extends Mapper<EppResourceIndex, Key<?>, Key<?>> {

18
java/google/registry/tools/server/RefreshDnsForAllDomainsAction.java
View file

@ -18,7 +18,6 @@ import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInp
import static google.registry.model.EppResourceUtils.isActive;
import static google.registry.model.registry.Registries.assertTldsExist;
import static google.registry.request.RequestParameters.PARAM_TLDS;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.annotations.VisibleForTesting;
@ -64,15 +63,14 @@ public class RefreshDnsForAllDomainsAction implements Runnable {
@Override
public void run() {
assertTldsExist(tlds);
response.sendJavaScriptRedirect(
createJobPath(
mrRunner
.setJobName("Refresh DNS for all domains")
.setModuleName("tools")
.setDefaultMapShards(10)
.runMapOnly(
new RefreshDnsForAllDomainsActionMapper(tlds),
ImmutableList.of(createEntityInput(DomainResource.class)))));
mrRunner
.setJobName("Refresh DNS for all domains")
.setModuleName("tools")
.setDefaultMapShards(10)
.runMapOnly(
new RefreshDnsForAllDomainsActionMapper(tlds),
ImmutableList.of(createEntityInput(DomainResource.class)))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to refresh DNS for all active domain resources. */

10
java/google/registry/tools/server/ResaveAllHistoryEntriesAction.java
View file

@ -15,7 +15,6 @@
package google.registry.tools.server;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
@ -51,14 +50,15 @@ public class ResaveAllHistoryEntriesAction implements Runnable {
@SuppressWarnings("unchecked")
@Override
public void run() {
response.sendJavaScriptRedirect(createJobPath(mrRunner
mrRunner
.setJobName("Re-save all HistoryEntry entities")
.setModuleName("tools")
.runMapOnly(
new ResaveAllHistoryEntriesActionMapper(),
ImmutableList.of(EppResourceInputs.createChildEntityInput(
ImmutableSet.of(EppResource.class),
ImmutableSet.of(HistoryEntry.class))))));
ImmutableList.of(
EppResourceInputs.createChildEntityInput(
ImmutableSet.of(EppResource.class), ImmutableSet.of(HistoryEntry.class))))
.sendLinkToMapreduceConsole(response);
}
/** Mapper to re-save all HistoryEntry entities. */

27
java/google/registry/util/PipelineUtils.java
View file

@ -1,27 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.util;
/** Utilities for working with pipeline jobs. */
public class PipelineUtils {
private static final String URL_PREFIX = "/_ah/pipeline/status.html?root=";
private PipelineUtils() {}
public static String createJobPath(String jobId) {
return URL_PREFIX + jobId;
}
}