zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-06-29 07:43:37 +02:00
Code Issues Projects Releases Packages Wiki Activity

Import code from internal repository to git

Browse source
This commit is contained in:
Justine Tunney 2016-03-01 17:18:14 -05:00
commit 0ef0c933d2
2490 changed files with 281594 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

30
java/com/google/domain/registry/tools/server/BUILD Normal file
View file

@ -0,0 +1,30 @@
package(
default_visibility = ["//java/com/google/domain/registry:registry_project"],
)
java_library(
name = "server",
srcs = glob(["**/*.java"]),
deps = [
"//java/com/google/common/base",
"//java/com/google/common/collect",
"//java/com/google/common/io",
"//java/com/google/domain/registry/config",
"//java/com/google/domain/registry/export",
"//java/com/google/domain/registry/gcs",
"//java/com/google/domain/registry/groups",
"//java/com/google/domain/registry/mapreduce",
"//java/com/google/domain/registry/model",
"//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/util",
"//third_party/java/appengine:appengine-api",
"//third_party/java/appengine_gcs_client",
"//third_party/java/appengine_mapreduce2:appengine_mapreduce",
"//third_party/java/dagger",
"//third_party/java/joda_time",
"//third_party/java/jsr305_annotations",
"//third_party/java/jsr330_inject",
"//third_party/java/objectify:objectify-v4_1",
],
)

130
java/com/google/domain/registry/tools/server/CreateGroupsTask.java Normal file
View file

@ -0,0 +1,130 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.domain.registry.export.SyncGroupMembersTask.getGroupEmailAddressForContactType;
import static com.google.domain.registry.request.Action.Method.POST;
import static java.util.Arrays.asList;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.groups.GroupsConnection;
import com.google.domain.registry.groups.GroupsConnection.Role;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.registrar.RegistrarContact;
import com.google.domain.registry.model.registrar.RegistrarContact.Type;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.BadRequestException;
import com.google.domain.registry.request.HttpException.InternalServerErrorException;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.Concurrent;
import com.google.domain.registry.util.FormattingLogger;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.List;
import javax.annotation.Nullable;
import javax.inject.Inject;
/** A task that creates Google Groups for a registrar's mailing lists. */
@Action(path = CreateGroupsTask.PATH, method = POST)
public class CreateGroupsTask implements Runnable {
public static final String PATH = "/_dr/admin/createGroups";
public static final String CLIENT_ID_PARAM = "clientId";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject GroupsConnection groupsConnection;
@Inject Response response;
@Inject @Config("publicDomainName") String publicDomainName;
@Inject @Parameter("clientId") Optional<String> clientId;
@Inject CreateGroupsTask() {}
@Override
public void run() {
final Registrar registrar = initAndLoadRegistrar();
if (registrar == null) {
return;
}
List<RegistrarContact.Type> types = asList(RegistrarContact.Type.values());
// Concurrently create the groups for each RegistrarContact.Type, collecting the results from
// each call (which are either an Exception if it failed, or absent() if it succeeded).
List<Optional<Exception>> results = Concurrent.transform(types,
new Function<RegistrarContact.Type, Optional<Exception>>() {
@Override
public Optional<Exception> apply(Type type) {
try {
String groupKey = getGroupEmailAddressForContactType(
registrar.getClientIdentifier(), type, publicDomainName);
String parentGroup =
getGroupEmailAddressForContactType("registrar", type, publicDomainName);
// Creates the group, then adds it as a member to the global registrar group for
// that type.
groupsConnection.createGroup(groupKey);
groupsConnection.addMemberToGroup(parentGroup, groupKey, Role.MEMBER);
return Optional.<Exception> absent();
} catch (Exception e) {
return Optional.of(e);
}
}});
// Return the correct server response based on the results of the group creations.
if (Optional.presentInstances(results).iterator().hasNext()) {
StringWriter responseString = new StringWriter();
PrintWriter responseWriter = new PrintWriter(responseString);
for (int i = 0; i < results.size(); i++) {
Optional<Exception> e = results.get(i);
if (e.isPresent()) {
responseWriter.append(types.get(i).getDisplayName()).append(" => ");
e.get().printStackTrace(responseWriter);
logger.severefmt(
e.get(),
"Could not create Google Group for registrar %s for type %s",
registrar.getRegistrarName(),
types.get(i).toString());
} else {
responseWriter.printf("%s => Success%n", types.get(i).getDisplayName());
}
}
throw new InternalServerErrorException(responseString.toString());
} else {
response.setStatus(SC_OK);
response.setPayload("Success!");
logger.info("Successfully created groups for registrar: " + registrar.getRegistrarName());
}
}
@Nullable
private Registrar initAndLoadRegistrar() {
if (!clientId.isPresent()) {
respondToBadRequest("Error creating Google Groups, missing parameter: clientId");
}
final Registrar registrar = Registrar.loadByClientId(clientId.get());
if (registrar == null) {
respondToBadRequest(String.format(
"Error creating Google Groups; could not find registrar with id %s", clientId.get()));
}
return registrar;
}
private void respondToBadRequest(String message) {
logger.severe(message);
throw new BadRequestException(message);
}
}

52
java/com/google/domain/registry/tools/server/CreateOrUpdatePremiumListAction.java Normal file
View file

@ -0,0 +1,52 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import com.google.common.collect.ImmutableMap;
import com.google.domain.registry.request.JsonResponse;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.util.FormattingLogger;
import javax.inject.Inject;
/**
* Abstract base class for actions that update premium lists.
*/
public abstract class CreateOrUpdatePremiumListAction implements Runnable {
protected static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
public static final String NAME_PARAM = "name";
public static final String INPUT_PARAM = "inputData";
@Inject JsonResponse response;
@Inject @Parameter(NAME_PARAM) String name;
@Inject @Parameter(INPUT_PARAM) String inputData;
@Override
public void run() {
try {
savePremiumList();
} catch (RuntimeException e) {
logger.severe(e, e.getMessage());
response.setPayload(ImmutableMap.of(
"error", e.toString(),
"status", "error"));
}
}
/** Creates a new premium list or updates an existing one. */
protected abstract void savePremiumList();
}

68
java/com/google/domain/registry/tools/server/CreatePremiumListAction.java Normal file
View file

@ -0,0 +1,68 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.domain.registry.model.registry.Registries.assertTldExists;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.domain.registry.model.registry.label.PremiumList;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Parameter;
import java.util.List;
import javax.inject.Inject;
/**
* An action that creates a premium list, for use by the registry_tool create_premium_list command.
*/
@Action(path = CreatePremiumListAction.PATH, method = POST)
public class CreatePremiumListAction extends CreateOrUpdatePremiumListAction {
public static final String OVERRIDE_PARAM = "override";
public static final String PATH = "/_dr/admin/createPremiumList";
@Inject @Parameter(OVERRIDE_PARAM) boolean override;
@Inject CreatePremiumListAction() {}
@Override
protected void savePremiumList() {
checkArgument(
!PremiumList.exists(name),
"A premium list of this name already exists: %s.", name);
if (!override) {
assertTldExists(name);
}
logger.infofmt("Saving premium list for TLD %s", name);
logger.infofmt("Got the following input data: %s", inputData);
List<String> inputDataPreProcessed =
Splitter.on('\n').omitEmptyStrings().splitToList(inputData);
PremiumList premiumList = new PremiumList.Builder()
.setName(name)
.setPremiumListMapFromLines(inputDataPreProcessed)
.build();
premiumList.saveAndUpdateEntries();
logger.infofmt("Saved premium list %s with entries %s",
premiumList.getName(),
premiumList.getPremiumListEntries());
response.setPayload(ImmutableMap.of("status", "success"));
}
}

118
java/com/google/domain/registry/tools/server/DeleteEntityAction.java Normal file
View file

@ -0,0 +1,118 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.googlecode.objectify.Key.create;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.common.base.Optional;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.BadRequestException;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.FormattingLogger;
import com.googlecode.objectify.VoidWork;
import com.googlecode.objectify.impl.EntityMetadata;
import javax.inject.Inject;
/**
* An action to delete entities in Datastore specified by raw key ids, which can be found in
* Datastore Viewer in the AppEngine console - it's the really long alphanumeric key that is
* labeled "Entity key" on the page for an individual entity.
*
* <p>rawKeys is the only required parameter. It is a comma-delimited list of Strings.
*
* <p><b>WARNING:</b> This servlet can be dangerous if used incorrectly as it can bypass checks on
* deletion (including whether the entity is referenced by other entities) and it does not write
* commit log entries for non-registered types. It should mainly be used for deleting testing or
* malformed data that cannot be properly deleted using existing tools. Generally, if there already
* exists an entity-specific deletion command, then use that one instead.
*/
@Action(path = DeleteEntityAction.PATH)
public class DeleteEntityAction implements Runnable {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
public static final String PATH = "/_dr/admin/deleteEntity";
public static final String PARAM_RAW_KEYS = "rawKeys";
@Inject @Parameter(PARAM_RAW_KEYS) String rawKeys;
@Inject Response response;
@Inject DeleteEntityAction() {}
@Override
public void run() {
// Get raw key strings from request
ImmutableList.Builder<Object> ofyDeletionsBuilder = new ImmutableList.Builder<>();
ImmutableList.Builder<Key> rawDeletionsBuilder = new ImmutableList.Builder<>();
for (String rawKeyString : Splitter.on(',').split(rawKeys)) {
// Convert raw keys string to real keys. Try to load it from Objectify if it's a registered
// type, and fall back to DatastoreService if its not registered.
Key rawKey = KeyFactory.stringToKey(rawKeyString);
Optional<Object> ofyEntity = loadOfyEntity(rawKey);
if (ofyEntity.isPresent()) {
ofyDeletionsBuilder.add(ofyEntity.get());
continue;
}
Optional<Entity> rawEntity = loadRawEntity(rawKey);
if (rawEntity.isPresent()) {
rawDeletionsBuilder.add(rawEntity.get().getKey());
continue;
}
// The entity could not be found by either Objectify or the datastore service
throw new BadRequestException("Could not find entity with key " + rawKeyString);
}
// Delete raw entities.
ImmutableList<Key> rawDeletions = rawDeletionsBuilder.build();
getDatastoreService().delete(rawDeletions);
// Delete ofy entities.
final ImmutableList<Object> ofyDeletions = ofyDeletionsBuilder.build();
ofy().transactNew(new VoidWork() {
@Override
public void vrun() {
ofy().delete().entities(ofyDeletions).now();
}});
String message = String.format(
"Deleted %d raw entities and %d registered entities",
rawDeletions.size(),
ofyDeletions.size());
logger.info(message);
response.setPayload(message);
}
private Optional<Object> loadOfyEntity(Key rawKey) {
EntityMetadata<Object> metadata = ofy().factory().getMetadata(rawKey.getKind());
return Optional.fromNullable(metadata == null ? null : ofy().load().key(create(rawKey)).now());
}
private Optional<Entity> loadRawEntity(Key rawKey) {
try {
return Optional.fromNullable(getDatastoreService().get(rawKey));
} catch (EntityNotFoundException e) {
logger.warningfmt(e, "Could not load entity from datastore service with key %s",
rawKey.toString());
return Optional.absent();
}
}
}

333
java/com/google/domain/registry/tools/server/GenerateZoneFilesAction.java Normal file
View file

@ -0,0 +1,333 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
import static com.google.common.base.Predicates.notNull;
import static com.google.common.collect.Iterables.transform;
import static com.google.common.collect.Iterators.filter;
import static com.google.common.io.BaseEncoding.base16;
import static com.google.domain.registry.mapreduce.EppResourceInputs.createEntityInput;
import static com.google.domain.registry.model.EppResourceUtils.loadAtPointInTime;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.request.Action.Method.POST;
import static com.google.domain.registry.util.PipelineUtils.createJobPath;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.joda.time.DateTimeZone.UTC;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.RetryParams;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.Reducer;
import com.google.appengine.tools.mapreduce.ReducerInput;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.gcs.GcsUtils;
import com.google.domain.registry.mapreduce.MapreduceRunner;
import com.google.domain.registry.mapreduce.NullInput;
import com.google.domain.registry.model.EppResource;
import com.google.domain.registry.model.domain.DomainResource;
import com.google.domain.registry.model.domain.ReferenceUnion;
import com.google.domain.registry.model.domain.secdns.DelegationSignerData;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.HttpException.BadRequestException;
import com.google.domain.registry.request.JsonActionRunner;
import com.google.domain.registry.util.Clock;
import com.googlecode.objectify.Ref;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.net.Inet4Address;
import java.net.InetAddress;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
/**
* MapReduce that requests generation of BIND zone files for a set of TLDs at a given time.
*
* <p>Zone files for each requested TLD are written to GCS. TLDs without entries produce zone files
* with only a header. The export time must be at least two minutes in the past and no more than
* 29 days in the past, and must be at midnight UTC.
*/
@Action(
path = GenerateZoneFilesAction.PATH,
method = POST,
xsrfProtection = true,
xsrfScope = "admin")
public class GenerateZoneFilesAction implements Runnable, JsonActionRunner.JsonAction {
public static final String PATH = "/_dr/task/generateZoneFiles";
/** Format for the zone file name. */
private static final String FILENAME_FORMAT = "%s-%s.zone";
/** Format for the GCS path to a file. */
private static final String GCS_PATH_FORMAT = "gs://%s/%s";
/** Format for the zone file header. */
private static final String HEADER_FORMAT = "$ORIGIN\t%s.\n\n";
/** Format for NS records. */
private static final String NS_FORMAT = "%s\t%d\tIN\tNS\t%s.\n";
/** Format for DS records. */
private static final String DS_FORMAT = "%s\t%d\tIN\tDS\t%d %d %d %s\n";
/** Format for A and AAAA records. */
private static final String A_FORMAT = "%s\t%d\tIN\t%s\t%s\n";
// TODO(b/20454352): Overhaul TTL configuration mechanism.
/** The time to live for exported NS record, in seconds. */
private static final int TTL_NS = 180;
/** The time to live for exported DS record, in seconds. */
private static final int TTL_DS = 86400;
/** The time to live for exported A/AAAA record, in seconds. */
private static final int TTL_A = 3600;
@Inject MapreduceRunner mrRunner;
@Inject JsonActionRunner jsonActionRunner;
@Inject @Config("zoneFilesBucket") String bucket;
@Inject @Config("gcsBufferSize") int gcsBufferSize;
@Inject @Config("commitLogDatastoreRetention") Duration datastoreRetention;
@Inject Clock clock;
@Inject GenerateZoneFilesAction() {}
@Override
public void run() {
jsonActionRunner.run(this);
}
@Override
public Map<String, Object> handleJsonRequest(Map<String, ?> json) {
@SuppressWarnings("unchecked")
ImmutableSet<String> tlds = ImmutableSet.copyOf((List<String>) json.get("tlds"));
final DateTime exportTime = DateTime.parse(json.get("exportTime").toString());
// We disallow exporting within the past 2 minutes because there might be outstanding writes.
// We can only reliably call loadAtPointInTime at times that are UTC midnight and >
// datastoreRetention ago in the past.
DateTime now = clock.nowUtc();
if (exportTime.isAfter(now.minusMinutes(2))) {
throw new BadRequestException("Invalid export time: must be > 2 minutes ago");
}
if (exportTime.isBefore(now.minus(datastoreRetention))) {
throw new BadRequestException(String.format(
"Invalid export time: must be < %d days ago",
datastoreRetention.getStandardDays()));
}
if (!exportTime.equals(exportTime.toDateTime(UTC).withTimeAtStartOfDay())) {
throw new BadRequestException("Invalid export time: must be midnight UTC");
}
String jobId = mrRunner
.setJobName("Generate bind file stanzas")
.setModuleName("tools")
.setDefaultReduceShards(tlds.size())
.runMapreduce(
new GenerateBindFileMapper(tlds, exportTime),
new GenerateBindFileReducer(bucket, exportTime, gcsBufferSize),
ImmutableList.of(
new NullInput<EppResource>(),
createEntityInput(DomainResource.class, HostResource.class)));
ImmutableList<String> filenames = FluentIterable.from(tlds)
.transform(
new Function<String, String>() {
@Override
public String apply(String tld) {
return String.format(
GCS_PATH_FORMAT,
bucket,
String.format(FILENAME_FORMAT, tld, exportTime));
}})
.toList();
return ImmutableMap.<String, Object>of(
"jobPath", createJobPath(jobId),
"filenames", filenames);
}
/** Mapper to find domains and hosts that were active at a given time. */
static class GenerateBindFileMapper extends Mapper<EppResource, String, String> {
private static final long serialVersionUID = 4647941823789859913L;
private final ImmutableSet<String> tlds;
private final DateTime exportTime;
GenerateBindFileMapper(ImmutableSet<String> tlds, DateTime exportTime) {
this.tlds = tlds;
this.exportTime = exportTime;
}
@Override
public void map(EppResource resource) {
if (resource == null) { // Force the reducer to always generate a bind header for each tld.
for (String tld : tlds) {
emit(tld, null);
}
} else if (resource instanceof DomainResource) {
mapDomain((DomainResource) resource);
} else {
mapHost((HostResource) resource);
}
}
private void mapDomain(DomainResource domain) {
// Domains never change their tld, so we can check if it's from the wrong tld right away.
if (tlds.contains(domain.getTld())) {
domain = loadAtPointInTime(domain, exportTime).now();
if (domain != null) { // A null means the domain was deleted (or not created) at this time.
String stanza = domainStanza(domain, exportTime);
if (!stanza.isEmpty()) {
emit(domain.getTld(), stanza);
getContext().incrementCounter(domain.getTld() + " domains");
}
}
}
}
private void mapHost(HostResource host) {
host = loadAtPointInTime(host, exportTime).now();
if (host != null) { // A null means the host was deleted (or not created) at this time.
// Find a matching tld. Hosts might change their tld, so check after the point-in-time load.
String fullyQualifiedHostName = host.getFullyQualifiedHostName();
for (String tld : tlds) {
if (fullyQualifiedHostName.endsWith("." + tld)) {
String stanza = hostStanza(host);
if (!stanza.isEmpty()) {
emit(tld, stanza);
getContext().incrementCounter(tld + " hosts");
}
return;
}
}
}
}
}
/** Reducer to write zone files to GCS. */
static class GenerateBindFileReducer extends Reducer<String, String, Void> {
private static final long serialVersionUID = -8489050680083119352L;
private final String bucket;
private final DateTime exportTime;
private final int gcsBufferSize;
GenerateBindFileReducer(String bucket, DateTime exportTime, int gcsBufferSize) {
this.bucket = bucket;
this.exportTime = exportTime;
this.gcsBufferSize = gcsBufferSize;
}
@Override
public void reduce(String tld, ReducerInput<String> stanzas) {
String stanzaCounter = tld + " stanzas";
GcsFilename filename =
new GcsFilename(bucket, String.format(FILENAME_FORMAT, tld, exportTime));
GcsUtils cloudStorage =
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8);
PrintWriter writer = new PrintWriter(osWriter)) {
writer.printf(HEADER_FORMAT, tld);
for (Iterator<String> stanzaIter = filter(stanzas, notNull()); stanzaIter.hasNext(); ) {
writer.println(stanzaIter.next());
getContext().incrementCounter(stanzaCounter);
}
writer.flush();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
/**
* Generates DNS records for a domain (NS and DS).
*
* These look like this:
* {@code
* foo.tld 180 IN NS ns.example.com.
* foo.tld 86400 IN DS 1 2 3 000102
* }
*/
private static String domainStanza(DomainResource domain, DateTime exportTime) {
StringBuilder result = new StringBuilder();
for (HostResource nameserver : ofy().load().refs(
transform(
domain.getNameservers(),
new Function<ReferenceUnion<HostResource>, Ref<HostResource>>() {
@Override
public Ref<HostResource> apply(ReferenceUnion<HostResource> referenceUnion) {
return referenceUnion.getLinked();
}})).values()) {
result.append(String.format(
NS_FORMAT,
domain.getFullyQualifiedDomainName(),
TTL_NS,
// Load the nameservers at the export time in case they've been renamed or deleted.
loadAtPointInTime(nameserver, exportTime).now().getFullyQualifiedHostName()));
}
for (DelegationSignerData dsData : domain.getDsData()) {
result.append(String.format(
DS_FORMAT,
domain.getFullyQualifiedDomainName(),
TTL_DS,
dsData.getKeyTag(),
dsData.getAlgorithm(),
dsData.getDigestType(),
base16().encode((dsData.getDigest()))));
}
return result.toString();
}
/**
* Generates DNS records for a domain (A and AAAA).
*
* <p>These look like this:
* {@code
* ns.foo.tld 3600 IN A 127.0.0.1
* ns.foo.tld 3600 IN AAAA 0:0:0:0:0:0:0:1
* }
*/
private static String hostStanza(HostResource host) {
StringBuilder result = new StringBuilder();
for (InetAddress addr : host.getInetAddresses()) {
// must be either IPv4 or IPv6
String rrSetClass = (addr instanceof Inet4Address) ? "A" : "AAAA";
result.append(String.format(
A_FORMAT,
host.getFullyQualifiedHostName(),
TTL_A,
rrSetClass,
addr.getHostAddress()));
}
return result.toString();
}
}

106
java/com/google/domain/registry/tools/server/KillAllCommitLogsAction.java Normal file
View file

@ -0,0 +1,106 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Iterables.partition;
import static com.google.common.collect.Lists.partition;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.request.Action.Method.POST;
import static com.google.domain.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Input;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.inputs.InMemoryInput;
import com.google.common.collect.ImmutableList;
import com.google.domain.registry.config.RegistryEnvironment;
import com.google.domain.registry.mapreduce.MapreduceAction;
import com.google.domain.registry.mapreduce.MapreduceRunner;
import com.google.domain.registry.model.ofy.CommitLogBucket;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Response;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.VoidWork;
import java.util.List;
import javax.inject.Inject;
/**
* Deletes all commit logs in datastore.
*
* <p>Before running this, use the datastore admin page to delete all {@code CommitLogManifest} and
* {@code CommitLogMutation} entities. That will take care of most (likely all) commit log entities
* (except perhaps for very recently created entities that are missed by the eventually consistent
* query driving that deletion) and it will be much faster than this mapreduce. After that, run this
* to get a guarantee that everything was deleted.
*/
@Action(path = "/_dr/task/killAllCommitLogs", method = POST)
public class KillAllCommitLogsAction implements MapreduceAction {
private static final int BATCH_SIZE = 100;
@Inject MapreduceRunner mrRunner;
@Inject Response response;
@Inject KillAllCommitLogsAction() {}
@Override
public void run() {
checkArgument( // safety
RegistryEnvironment.get() == RegistryEnvironment.ALPHA
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
"DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA OR TESTS.");
// Create a in-memory input, assigning each bucket to its own shard for maximum parallelization.
Input<Key<CommitLogBucket>> input =
new InMemoryInput<>(partition(CommitLogBucket.getAllBucketKeys().asList(), 1));
response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Delete all commit logs")
.setModuleName("tools")
.runMapOnly(new KillAllCommitLogsMapper(), ImmutableList.of(input))));
}
/**
* Mapper to delete a {@link CommitLogBucket} and any commit logs in the bucket.
*
* <p>This will delete:
* <ul>
* <li>{@link CommitLogBucket}
* <li>{@code CommitLogManifest}
* <li>{@code CommitLogMutation}
* </ul>
*/
static class KillAllCommitLogsMapper extends Mapper<Key<CommitLogBucket>, Void, Void> {
private static final long serialVersionUID = 1504266335352952033L;
@Override
public void map(Key<CommitLogBucket> bucket) {
// The query on the bucket could time out, but we are not worried about that because of the
// procedure outlined above.
for (final List<Key<Object>> batch
: partition(ofy().load().ancestor(bucket).keys(), BATCH_SIZE)) {
ofy().transact(new VoidWork() {
@Override
public void vrun() {
ofy().deleteWithoutBackup().entities(batch);
}});
getContext().incrementCounter("deleted entities", batch.size());
}
getContext().incrementCounter("completed buckets");
}
}
}

110
java/com/google/domain/registry/tools/server/KillAllEppResourcesAction.java Normal file
View file

@ -0,0 +1,110 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Iterables.partition;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.request.Action.Method.POST;
import static com.google.domain.registry.util.PipelineUtils.createJobPath;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
import com.google.domain.registry.config.RegistryEnvironment;
import com.google.domain.registry.mapreduce.EppResourceInputs;
import com.google.domain.registry.mapreduce.MapreduceAction;
import com.google.domain.registry.mapreduce.MapreduceRunner;
import com.google.domain.registry.model.EppResource;
import com.google.domain.registry.model.domain.DomainApplication;
import com.google.domain.registry.model.index.DomainApplicationIndex;
import com.google.domain.registry.model.index.EppResourceIndex;
import com.google.domain.registry.model.index.ForeignKeyIndex;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Response;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.VoidWork;
import java.util.List;
import javax.inject.Inject;
/** Deletes all {@link EppResource} objects in datastore, including indices and descendants. */
@Action(path = "/_dr/task/killAllEppResources", method = POST)
public class KillAllEppResourcesAction implements MapreduceAction {
private static final int BATCH_SIZE = 100;
@Inject MapreduceRunner mrRunner;
@Inject Response response;
@Inject KillAllEppResourcesAction() {}
@Override
public void run() {
checkArgument( // safety
RegistryEnvironment.get() == RegistryEnvironment.ALPHA
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
"DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA OR TESTS.");
response.sendJavaScriptRedirect(createJobPath(mrRunner
.setJobName("Delete all EppResources, children, and indices")
.setModuleName("tools")
.runMapOnly(
new KillAllEppResourcesMapper(),
ImmutableList.of(EppResourceInputs.createIndexInput()))));
}
static class KillAllEppResourcesMapper extends Mapper<EppResourceIndex, Void, Void> {
private static final long serialVersionUID = 103826288518612669L;
/**
* Delete an {@link EppResourceIndex}, its referent, all descendants of each referent, and the
* {@link ForeignKeyIndex} or {@link DomainApplicationIndex} of the referent, as appropriate.
*
* <p>This will delete:
* <ul>
* <li>All {@link ForeignKeyIndex} types
* <li>{@link DomainApplicationIndex}
* <li>{@link EppResourceIndex}
* <li>All {@link EppResource} types
* <li>{@code HistoryEntry}
* <li>All {@code BillingEvent} types
* <li>All {@code PollMessage} types
* </ul>
*/
@Override
public void map(final EppResourceIndex eri) {
EppResource resource = eri.getReference().get();
for (final List<Key<Object>> batch
: partition(ofy().load().ancestor(resource).keys(), BATCH_SIZE)) {
ofy().transact(new VoidWork() {
@Override
public void vrun() {
ofy().deleteWithoutBackup().entities(batch);
}});
getContext().incrementCounter("deleted descendants", batch.size());
}
final Key<?> foreignKey = resource instanceof DomainApplication
? DomainApplicationIndex.createKey((DomainApplication) resource)
: ForeignKeyIndex.createKey(resource);
ofy().transact(new VoidWork() {
@Override
public void vrun() {
ofy().deleteWithoutBackup().keys(Key.create(eri), foreignKey).now();
}});
getContext().incrementCounter("deleted eris");
}
}
}

59
java/com/google/domain/registry/tools/server/ListDomainsAction.java Normal file
View file

@ -0,0 +1,59 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.domain.registry.model.EppResourceUtils.queryNotDeleted;
import static com.google.domain.registry.model.registry.Registries.assertTldExists;
import static com.google.domain.registry.request.Action.Method.GET;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.domain.DomainResource;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.util.Clock;
import java.util.Comparator;
import javax.inject.Inject;
/** An action that lists domains, for use by the registry_tool list_domains command. */
@Action(path = ListDomainsAction.PATH, method = {GET, POST})
public final class ListDomainsAction extends ListObjectsAction<DomainResource> {
public static final String PATH = "/_dr/admin/list/domains";
public static final String TLD_PARAM = "tld";
@Inject @Parameter("tld") String tld;
@Inject Clock clock;
@Inject ListDomainsAction() {}
@Override
public ImmutableSet<String> getPrimaryKeyFields() {
return ImmutableSet.of("fullyQualifiedDomainName");
}
@Override
public ImmutableSet<DomainResource> loadObjects() {
return FluentIterable
.from(queryNotDeleted(DomainResource.class, clock.nowUtc(), "tld", assertTldExists(tld)))
.toSortedSet(new Comparator<DomainResource>() {
@Override
public int compare(DomainResource a, DomainResource b) {
return a.getFullyQualifiedDomainName().compareTo(b.getFullyQualifiedDomainName());
}});
}
}

69
java/com/google/domain/registry/tools/server/ListHostsAction.java Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.request.Action.Method.GET;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.EppResourceUtils;
import com.google.domain.registry.model.host.HostResource;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.util.Clock;
import org.joda.time.DateTime;
import java.util.Comparator;
import javax.inject.Inject;
/** An action that lists hosts, for use by the registry_tool list_hosts command. */
@Action(path = ListHostsAction.PATH, method = {GET, POST})
public final class ListHostsAction extends ListObjectsAction<HostResource> {
public static final String PATH = "/_dr/admin/list/hosts";
private static final Comparator<HostResource> comparator =
new Comparator<HostResource>() {
@Override
public int compare(HostResource host1, HostResource host2) {
return host1.getFullyQualifiedHostName()
.compareTo(host2.getFullyQualifiedHostName());
}};
@Inject Clock clock;
@Inject ListHostsAction() {}
@Override
public ImmutableSet<String> getPrimaryKeyFields() {
return ImmutableSet.of("fullyQualifiedHostName");
}
@Override
public ImmutableSet<HostResource> loadObjects() {
final DateTime now = clock.nowUtc();
return FluentIterable
.from(ofy().load().type(HostResource.class))
.filter(new Predicate<HostResource>() {
@Override
public boolean apply(HostResource host) {
return EppResourceUtils.isActive(host, now);
}})
.toSortedSet(comparator);
}
}

286
java/com/google/domain/registry/tools/server/ListObjectsAction.java Normal file
View file

@ -0,0 +1,286 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.ImmutableTable;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Ordering;
import com.google.domain.registry.model.ImmutableObject;
import com.google.domain.registry.request.JsonResponse;
import com.google.domain.registry.request.Parameter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import javax.inject.Inject;
/**
* Abstract base class for actions that list ImmutableObjects.
*
* <p>Returns formatted text to be displayed on the screen.
*
* @param <T> type of object
*/
public abstract class ListObjectsAction<T extends ImmutableObject> implements Runnable {
public static final String FIELDS_PARAM = "fields";
public static final String PRINT_HEADER_ROW_PARAM = "printHeaderRow";
public static final String FULL_FIELD_NAMES_PARAM = "fullFieldNames";
@Inject JsonResponse response;
@Inject @Parameter("fields") Optional<String> fields;
@Inject @Parameter("printHeaderRow") Optional<Boolean> printHeaderRow;
@Inject @Parameter("fullFieldNames") Optional<Boolean> fullFieldNames;
/** Returns the set of objects to list, in the desired listing order. */
abstract ImmutableSet<T> loadObjects();
/**
* Returns a set of fields to always include in the output as the leftmost columns. Subclasses
* can use this to specify the equivalent of a "primary key" for each object listed.
*/
ImmutableSet<String> getPrimaryKeyFields() {
return ImmutableSet.of();
}
/**
* Returns an {@link ImmutableBiMap} that maps any field name aliases to the actual field names.
* <p>
* Users can select aliased fields for display using either the original name or the alias. By
* default, aliased fields will use the alias name as the header instead of the original name.
*/
ImmutableBiMap<String, String> getFieldAliases() {
return ImmutableBiMap.of();
}
/**
* Returns for a given {@link ImmutableObject} a mapping from field names to field values that
* will override, for any overlapping field names, the default behavior of getting the field
* value by looking up that field name in the map returned by
* {@link ImmutableObject#toDiffableFieldMap}.
* <p>
* This can be used to specify customized printing of certain fields (e.g. to print out a boolean
* field as "active" or "-" instead of "true" or "false"). It can also be used to add fields to
* the data, e.g. for computed fields that can be accessed from the object directly but aren't
* stored as simple fields.
*/
ImmutableMap<String, String> getFieldOverrides(@SuppressWarnings("unused") T object) {
return ImmutableMap.of();
}
@Override
public void run() {
try {
// Get the object data first, so we can figure out the list of all available fields using the
// data if necessary.
ImmutableSet<T> objects = loadObjects();
// Get the list of fields we should return.
ImmutableSet<String> fieldsToUse = getFieldsToUse(objects);
// Convert the data into a table.
ImmutableTable<T, String, String> data = extractData(fieldsToUse, objects);
// Now that we have the data table, compute the column widths.
ImmutableMap<String, Integer> columnWidths =
computeColumnWidths(data, isHeaderRowInUse(data));
// Finally, convert the table to an array of lines of text.
List<String> lines = generateFormattedData(data, columnWidths);
// Return the results.
response.setPayload(ImmutableMap.of(
"lines", lines,
"status", "success"));
} catch (Exception e) {
String message = e.getMessage();
if (message == null) {
message = e.getClass().getName();
}
response.setPayload(ImmutableMap.of(
"error", message,
"status", "error"));
}
}
/**
* Returns the set of fields to return, aliased or not according to --full_field_names, and
* with duplicates eliminated but the ordering otherwise preserved.
*/
private ImmutableSet<String> getFieldsToUse(ImmutableSet<T> objects) {
// Get the list of fields from the received parameter.
List<String> fieldsToUse;
if ((fields == null) || !fields.isPresent()) {
fieldsToUse = new ArrayList<>();
} else {
fieldsToUse = Splitter.on(',').splitToList(fields.get());
// Check whether any field name is the wildcard; if so, use all fields.
if (fieldsToUse.contains("*")) {
fieldsToUse = getAllAvailableFields(objects);
}
}
// Handle aliases according to the state of the fullFieldNames parameter.
final ImmutableMap<String, String> nameMapping =
((fullFieldNames != null) && fullFieldNames.isPresent() && fullFieldNames.get())
? getFieldAliases() : getFieldAliases().inverse();
return ImmutableSet.copyOf(Iterables.transform(
Iterables.concat(getPrimaryKeyFields(), fieldsToUse),
new Function<String, String>() {
@Override
public String apply(String field) {
// Rename fields that are in the map according to the map, and leave the others as is.
return nameMapping.containsKey(field) ? nameMapping.get(field) : field;
}}));
}
/**
* Constructs a list of all available fields for use by the wildcard field specification.
* Don't include aliases, since then we'd wind up returning the same field twice.
*/
private ImmutableList<String> getAllAvailableFields(ImmutableSet<T> objects) {
ImmutableList.Builder<String> fields = new ImmutableList.Builder<>();
for (T object : objects) {
// Base case of the mapping is to use ImmutableObject's toDiffableFieldMap().
fields.addAll(object.toDiffableFieldMap().keySet());
// Next, overlay any field-level overrides specified by the subclass.
fields.addAll(getFieldOverrides(object).keySet());
}
return fields.build();
}
/**
* Returns a table of data for the given sets of fields and objects. The table is row-keyed by
* object and column-keyed by field, in the same iteration order as the provided sets.
*/
private ImmutableTable<T, String, String>
extractData(ImmutableSet<String> fields, ImmutableSet<T> objects) {
ImmutableTable.Builder<T, String, String> builder = ImmutableTable.builder();
for (T object : objects) {
Map<String, Object> fieldMap = new HashMap<>();
// Base case of the mapping is to use ImmutableObject's toDiffableFieldMap().
fieldMap.putAll(object.toDiffableFieldMap());
// Next, overlay any field-level overrides specified by the subclass.
fieldMap.putAll(getFieldOverrides(object));
// Next, add to the mapping all the aliases, with their values defined as whatever was in the
// map under the aliased field's original name.
fieldMap.putAll(
Maps.transformValues(getFieldAliases(), Functions.forMap(new HashMap<>(fieldMap))));
Set<String> expectedFields = ImmutableSortedSet.copyOf(fieldMap.keySet());
for (String field : fields) {
checkArgument(fieldMap.containsKey(field),
"Field '%s' not found - recognized fields are:\n%s", field, expectedFields);
builder.put(object, field, Objects.toString(fieldMap.get(field), ""));
}
}
return builder.build();
}
/**
* Computes the column widths of the given table of strings column-keyed by strings and returns
* them as a map from column key name to integer width. The column width is defined as the max
* length of any string in that column, including the name of the column.
*/
private static ImmutableMap<String, Integer> computeColumnWidths(
ImmutableTable<?, String, String> data, final boolean includingHeader) {
return ImmutableMap.copyOf(Maps.transformEntries(
data.columnMap(),
new Maps.EntryTransformer<String, Map<?, String>, Integer>() {
@Override
public Integer transformEntry(String columnName, Map<?, String> columnValues) {
// Return the length of the longest string in this column (including the column name).
return Ordering.natural().max(Iterables.transform(
Iterables.concat(
ImmutableList.of(includingHeader ? columnName : ""),
columnValues.values()),
new Function<String, Integer>() {
@Override
public Integer apply(String value) {
return value.length();
}}));
}}));
}
/**
* Check whether to display headers. If the parameter is not set, print headers only if there
* is more than one column.
*/
private boolean isHeaderRowInUse(final ImmutableTable<?, String, String> data) {
return ((printHeaderRow != null) && printHeaderRow.isPresent())
? printHeaderRow.get() : (data.columnKeySet().size() > 1);
}
/** Converts the provided table of data to text, formatted using the provided column widths. */
private List<String> generateFormattedData(
ImmutableTable<T, String, String> data,
ImmutableMap<String, Integer> columnWidths) {
Function<Map<String, String>, String> rowFormatter = makeRowFormatter(columnWidths);
List<String> lines = new ArrayList<>();
if (isHeaderRowInUse(data)) {
// Add a row of headers (column names mapping to themselves).
Map<String, String> headerRow =
Maps.asMap(data.columnKeySet(), Functions.<String>identity());
lines.add(rowFormatter.apply(headerRow));
// Add a row of separator lines (column names mapping to '-' * column width).
Map<String, String> separatorRow = Maps.transformValues(columnWidths,
new Function<Integer, String>() {
@Override
public String apply(Integer width) {
return Strings.repeat("-", width);
}});
lines.add(rowFormatter.apply(separatorRow));
}
// Add the actual data rows.
for (Map<String, String> row : data.rowMap().values()) {
lines.add(rowFormatter.apply(row));
}
return lines;
}
/**
* Returns for the given column widths map a row formatting function that converts a row map (of
* column keys to cell values) into a single string with each column right-padded to that width.
* <p>
* The resulting strings separate padded fields with two spaces and each end in a newline.
*/
private static Function<Map<String, String>, String> makeRowFormatter(
final Map<String, Integer> columnWidths) {
return new Function<Map<String, String>, String>() {
@Override
public String apply(Map<String, String> rowByColumns) {
List<String> paddedFields = new ArrayList<>();
for (Map.Entry<String, String> cell : rowByColumns.entrySet()) {
paddedFields.add(Strings.padEnd(cell.getValue(), columnWidths.get(cell.getKey()), ' '));
}
return Joiner.on(" ").join(paddedFields);
}};
}
}

46
java/com/google/domain/registry/tools/server/ListPremiumListsAction.java Normal file
View file

@ -0,0 +1,46 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.domain.registry.model.common.EntityGroupRoot.getCrossTldKey;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.request.Action.Method.GET;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.registry.label.PremiumList;
import com.google.domain.registry.request.Action;
import javax.inject.Inject;
/** An action that lists premium lists, for use by the registry_tool list_premium_lists command. */
@Action(path = ListPremiumListsAction.PATH, method = {GET, POST})
public final class ListPremiumListsAction extends ListObjectsAction<PremiumList> {
public static final String PATH = "/_dr/admin/list/premiumLists";
@Inject ListPremiumListsAction() {}
@Override
public ImmutableSet<String> getPrimaryKeyFields() {
return ImmutableSet.of("name");
}
@Override
public ImmutableSet<PremiumList> loadObjects() {
return ImmutableSet.copyOf(
ofy().load().type(PremiumList.class).ancestor(getCrossTldKey()).list());
}
}

58
java/com/google/domain/registry/tools/server/ListRegistrarsAction.java Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.domain.registry.request.Action.Method.GET;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.request.Action;
import javax.inject.Inject;
/** An action that lists registrars, for use by the registry_tool list_registrars command. */
@Action(path = ListRegistrarsAction.PATH, method = {GET, POST})
public final class ListRegistrarsAction extends ListObjectsAction<Registrar> {
public static final String PATH = "/_dr/admin/list/registrars";
@Inject ListRegistrarsAction() {}
@Override
public ImmutableSet<String> getPrimaryKeyFields() {
return ImmutableSet.of("clientIdentifier");
}
@Override
public ImmutableSet<Registrar> loadObjects() {
return ImmutableSet.copyOf(Registrar.loadAll());
}
@Override
public ImmutableBiMap<String, String> getFieldAliases() {
return ImmutableBiMap.of(
"clientId", "clientIdentifier",
"premiumNames", "blockPremiumNames");
}
@Override
public ImmutableMap<String, String> getFieldOverrides(Registrar registrar) {
return ImmutableMap.of(
"blockPremiumNames", registrar.getBlockPremiumNames() ? "blocked" : "-");
}
}

46
java/com/google/domain/registry/tools/server/ListReservedListsAction.java Normal file
View file

@ -0,0 +1,46 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.domain.registry.model.common.EntityGroupRoot.getCrossTldKey;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import static com.google.domain.registry.request.Action.Method.GET;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.registry.label.ReservedList;
import com.google.domain.registry.request.Action;
import javax.inject.Inject;
/** A that lists reserved lists, for use by the registry_tool list_reserved_lists command. */
@Action(path = ListReservedListsAction.PATH, method = {GET, POST})
public final class ListReservedListsAction extends ListObjectsAction<ReservedList> {
public static final String PATH = "/_dr/admin/list/reservedLists";
@Inject ListReservedListsAction() {}
@Override
public ImmutableSet<String> getPrimaryKeyFields() {
return ImmutableSet.of("name");
}
@Override
public ImmutableSet<ReservedList> loadObjects() {
return ImmutableSet.copyOf(
ofy().load().type(ReservedList.class).ancestor(getCrossTldKey()).list());
}
}

82
java/com/google/domain/registry/tools/server/ListTldsAction.java Normal file
View file

@ -0,0 +1,82 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.domain.registry.model.registry.Registries.getTlds;
import static com.google.domain.registry.request.Action.Method.GET;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.model.registry.Registry;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.util.Clock;
import org.joda.time.DateTime;
import javax.inject.Inject;
/** An action that lists top-level domains, for use by the registry_tool list_tlds command. */
@Action(path = ListTldsAction.PATH, method = {GET, POST})
public final class ListTldsAction extends ListObjectsAction<Registry> {
public static final String PATH = "/_dr/admin/list/tlds";
@Inject Clock clock;
@Inject ListTldsAction() {}
@Override
public ImmutableSet<String> getPrimaryKeyFields() {
return ImmutableSet.of("tldStr");
}
@Override
public ImmutableSet<Registry> loadObjects() {
return FluentIterable.from(getTlds())
.transform(new Function<String, Registry>() {
@Override
public Registry apply(String tldString) {
return Registry.get(tldString);
}})
.toSet();
}
@Override
public ImmutableBiMap<String, String> getFieldAliases() {
return ImmutableBiMap.of(
"TLD", "tldStr",
"dns", "dnsPaused",
"escrow", "escrowEnabled",
"premiumPricing", "premiumPriceAckRequired");
}
@Override
public ImmutableMap<String, String> getFieldOverrides(Registry registry) {
final DateTime now = clock.nowUtc();
return new ImmutableMap.Builder<String, String>()
.put("dnsPaused", registry.getDnsPaused() ? "paused" : "-")
.put("escrowEnabled", registry.getEscrowEnabled() ? "enabled" : "-")
.put("premiumPriceAckRequired", registry.getPremiumPriceAckRequired() ? "ack req'd" : "-")
.put("tldState", registry.isPdt(now) ? "PDT" : registry.getTldState(now).toString())
.put("tldStateTransitions", registry.getTldStateTransitions().toString())
.put("renewBillingCost", registry.getStandardRenewCost(now).toString())
.put("renewBillingCostTransitions", registry.getRenewBillingCostTransitions().toString())
.build();
}
}

91
java/com/google/domain/registry/tools/server/ToolsServerModule.java Normal file
View file

@ -0,0 +1,91 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.common.base.Strings.emptyToNull;
import static com.google.domain.registry.request.RequestParameters.extractBooleanParameter;
import static com.google.domain.registry.request.RequestParameters.extractOptionalParameter;
import static com.google.domain.registry.request.RequestParameters.extractRequiredParameter;
import com.google.common.base.Optional;
import com.google.domain.registry.request.Parameter;
import dagger.Module;
import dagger.Provides;
import javax.servlet.http.HttpServletRequest;
/**
* Dagger module for the tools package.
*/
@Module
public class ToolsServerModule {
@Provides
@Parameter("clientId")
static Optional<String> provideClientId(HttpServletRequest req) {
return Optional.fromNullable(emptyToNull(req.getParameter(CreateGroupsTask.CLIENT_ID_PARAM)));
}
@Provides
@Parameter("fields")
static Optional<String> provideFields(HttpServletRequest req) {
return extractOptionalParameter(req, ListObjectsAction.FIELDS_PARAM);
}
@Provides
@Parameter("fullFieldNames")
static Optional<Boolean> provideFullFieldNames(HttpServletRequest req) {
String s = emptyToNull(req.getParameter(ListObjectsAction.FULL_FIELD_NAMES_PARAM));
return (s == null) ? Optional.<Boolean>absent() : Optional.of(Boolean.parseBoolean(s));
}
@Provides
@Parameter("inputData")
static String provideInput(HttpServletRequest req) {
return extractRequiredParameter(req, CreatePremiumListAction.INPUT_PARAM);
}
@Provides
@Parameter("name")
static String provideName(HttpServletRequest req) {
return extractRequiredParameter(req, CreatePremiumListAction.NAME_PARAM);
}
@Provides
@Parameter("override")
static boolean provideOverride(HttpServletRequest req) {
return extractBooleanParameter(req, CreatePremiumListAction.OVERRIDE_PARAM);
}
@Provides
@Parameter("printHeaderRow")
static Optional<Boolean> providePrintHeaderRow(HttpServletRequest req) {
String s = emptyToNull(req.getParameter(ListObjectsAction.PRINT_HEADER_ROW_PARAM));
return (s == null) ? Optional.<Boolean>absent() : Optional.of(Boolean.parseBoolean(s));
}
@Provides
@Parameter("tld")
static String provideTld(HttpServletRequest req) {
return extractRequiredParameter(req, "tld");
}
@Provides
@Parameter("rawKeys")
static String provideRawKeys(HttpServletRequest req) {
return extractRequiredParameter(req, "rawKeys");
}
}

69
java/com/google/domain/registry/tools/server/UpdatePremiumListAction.java Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.domain.registry.request.Action.Method.POST;
import com.google.common.base.Optional;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.domain.registry.model.registry.label.PremiumList;
import com.google.domain.registry.request.Action;
import java.util.List;
import javax.inject.Inject;
/**
* An action that creates a premium list, for use by the registry_tool create_premium_list command.
*/
@Action(path = UpdatePremiumListAction.PATH, method = POST)
public class UpdatePremiumListAction extends CreateOrUpdatePremiumListAction {
public static final String PATH = "/_dr/admin/updatePremiumList";
@Inject UpdatePremiumListAction() {}
@Override
protected void savePremiumList() {
Optional<PremiumList> existingName = PremiumList.get(name);
checkArgument(
existingName.isPresent(),
"Could not update premium list %s because it doesn't exist.",
name);
logger.infofmt("Updating premium list for TLD %s", name);
logger.infofmt("Got the following input data: %s", inputData);
List<String> inputDataPreProcessed =
Splitter.on('\n').omitEmptyStrings().splitToList(inputData);
PremiumList premiumList = existingName.get().asBuilder()
.setPremiumListMapFromLines(inputDataPreProcessed)
.build();
premiumList.saveAndUpdateEntries();
logger.infofmt("Updated premium list %s with entries %s",
premiumList.getName(),
premiumList.getPremiumListEntries());
String message = String.format(
"Saved premium list %s with %d entries.\n",
premiumList.getName(),
premiumList.getPremiumListEntries().size());
response.setPayload(ImmutableMap.of(
"status", "success",
"message", message));
}
}

136
java/com/google/domain/registry/tools/server/javascrap/AnnihilateNonDefaultNamespacesAction.java Normal file
View file

@ -0,0 +1,136 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.tools.server.javascrap;
import static com.google.appengine.api.datastore.DatastoreServiceFactory.getDatastoreService;
import static com.google.common.base.Preconditions.checkState;
import static com.google.domain.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
import static com.google.domain.registry.model.ofy.ObjectifyService.ofy;
import com.google.appengine.api.NamespaceManager;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.Entities;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.tools.mapreduce.Input;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.inputs.DatastoreKeyInput;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.domain.registry.mapreduce.ChunkingKeyInput;
import com.google.domain.registry.mapreduce.MapreduceAction;
import com.google.domain.registry.mapreduce.MapreduceRunner;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.PipelineUtils;
import com.googlecode.objectify.VoidWork;
import java.util.List;
import javax.inject.Inject;
/**
* A mapreduce that deletes all entities in all namespaces except for the default namespace.
*/
@Action(path = "/_dr/task/annihilateNonDefaultNamespaces")
public class AnnihilateNonDefaultNamespacesAction implements MapreduceAction {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@Inject DatastoreService datastoreService;
@Inject @Parameter(PARAM_DRY_RUN) boolean isDryRun;
@Inject MapreduceRunner mrRunner;
@Inject Response response;
@Inject AnnihilateNonDefaultNamespacesAction() {}
@Override
public void run() {
response.sendJavaScriptRedirect(PipelineUtils.createJobPath(mrRunner
.setJobName("Annihilate non-default namespaces")
.setModuleName("backend")
.runMapOnly(
new AnnihilateNonDefaultNamespacesMapper(isDryRun),
getInputs())));
}
/** Mapper to delete all entities in non-default namespaces. */
public static class AnnihilateNonDefaultNamespacesMapper extends Mapper<List<Key>, Void, Void> {
private static final long serialVersionUID = 8415923063881853727L;
private final boolean isDryRun;
public AnnihilateNonDefaultNamespacesMapper(boolean isDryRun) {
this.isDryRun = isDryRun;
}
@Override
public final void map(final List<Key> keys) {
if (isDryRun) {
logger.infofmt("Would delete these entities: %s", keys);
} else {
for (Key key : keys) {
// Additional safety check to prevent deleting real data.
checkState(
!Strings.isNullOrEmpty(key.getNamespace()),
"Will not delete key %s is in default namespace",
key);
}
ofy().transact(new VoidWork() {
@Override
public void vrun() {
getDatastoreService().delete(keys);
}});
}
getContext().incrementCounter("entities deleted", keys.size());
}
}
private Iterable<Input<List<Key>>> getInputs() {
ImmutableSet.Builder<String> namespaces = new ImmutableSet.Builder<>();
Query namespacesQuery = new Query(Entities.NAMESPACE_METADATA_KIND).setKeysOnly();
for (Entity entity : datastoreService.prepare(namespacesQuery).asIterable()) {
// Don't delete anything in the default namespace!
if (!Strings.isNullOrEmpty(entity.getKey().getName())) {
namespaces.add(entity.getKey().getName());
}
}
ImmutableSet.Builder<Input<List<Key>>> inputs = new ImmutableSet.Builder<>();
for (String namespace : namespaces.build()) {
NamespaceManager.set(namespace);
ImmutableSet.Builder<String> kindsBuilder = new ImmutableSet.Builder<>();
Query kindsQuery = new Query(Entities.KIND_METADATA_KIND).setKeysOnly();
for (Entity entity : datastoreService.prepare(kindsQuery).asIterable()) {
// Don't delete built-in kinds such as __Stat_* entities.
if (!entity.getKey().getName().startsWith("_")) {
kindsBuilder.add(entity.getKey().getName());
}
}
ImmutableSet<String> kinds = kindsBuilder.build();
logger.infofmt("For namespace %s, found kinds: %s", namespace, kinds);
for (String kind : kinds) {
// Don't try to parallelize here, because Registry 1.0 entities are almost all in a single
// entity group.
inputs.add(new ChunkingKeyInput(new DatastoreKeyInput(kind, 1, namespace), 20));
}
}
NamespaceManager.set("");
return inputs.build();
}
}