mirror of
https://github.com/google/nomulus.git
synced 2025-05-14 00:17:20 +02:00
Add mapreduce for RDE hosts import
------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=136766682
This commit is contained in:
parent
975f574256
commit
4cfe107475
22 changed files with 3654 additions and 0 deletions
114
java/google/registry/rde/RdeHostImportAction.java
Normal file
114
java/google/registry/rde/RdeHostImportAction.java
Normal file
|
@ -0,0 +1,114 @@
|
|||
// Copyright 2016 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.rde;
|
||||
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.rde.RdeModule.PATH;
|
||||
import static google.registry.util.PipelineUtils.createJobPath;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
import com.google.appengine.tools.mapreduce.Mapper;
|
||||
import com.google.common.base.Optional;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.config.ConfigModule;
|
||||
import google.registry.config.ConfigModule.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.util.SystemClock;
|
||||
import javax.inject.Inject;
|
||||
|
||||
/**
|
||||
* A mapreduce that imports hosts from an escrow file.
|
||||
*
|
||||
* <p>Specify the escrow file to import with the "path" parameter.
|
||||
*/
|
||||
@Action(path = "/_dr/task/importRdeHosts")
|
||||
public class RdeHostImportAction implements Runnable {
|
||||
|
||||
private static final GcsService GCS_SERVICE =
|
||||
GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
|
||||
|
||||
private final MapreduceRunner mrRunner;
|
||||
private final Response response;
|
||||
private final String importBucketName;
|
||||
private final String importFileName;
|
||||
private final Optional<Integer> mapShards;
|
||||
|
||||
@Inject
|
||||
public RdeHostImportAction(
|
||||
MapreduceRunner mrRunner,
|
||||
Response response,
|
||||
@Config("rdeImportBucket") String importBucketName,
|
||||
@Parameter(PATH) String importFileName,
|
||||
@Parameter(PARAM_MAP_SHARDS) Optional<Integer> mapShards) {
|
||||
this.mrRunner = mrRunner;
|
||||
this.response = response;
|
||||
this.importBucketName = importBucketName;
|
||||
this.importFileName = importFileName;
|
||||
this.mapShards = mapShards;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
response.sendJavaScriptRedirect(createJobPath(mrRunner
|
||||
.setJobName("Import hosts from escrow file")
|
||||
.setModuleName("backend")
|
||||
.runMapOnly(
|
||||
new RdeHostImportMapper(importBucketName),
|
||||
ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))));
|
||||
}
|
||||
|
||||
/** Mapper to import hosts from an escrow file. */
|
||||
public static class RdeHostImportMapper extends Mapper<HostResource, Void, Void> {
|
||||
|
||||
private static final long serialVersionUID = -2898753709127134419L;
|
||||
private final String importBucketName;
|
||||
private transient RdeImportUtils importUtils;
|
||||
|
||||
public RdeHostImportMapper(String importBucketName) {
|
||||
this.importBucketName = importBucketName;
|
||||
}
|
||||
|
||||
private RdeImportUtils getImportUtils() {
|
||||
if (importUtils == null) {
|
||||
importUtils = createRdeImportUtils();
|
||||
}
|
||||
return importUtils;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new instance of RdeImportUtils.
|
||||
*/
|
||||
private RdeImportUtils createRdeImportUtils() {
|
||||
return new RdeImportUtils(
|
||||
ofy(),
|
||||
new SystemClock(),
|
||||
importBucketName,
|
||||
new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void map(HostResource host) {
|
||||
getImportUtils().importHost(host);
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue