mv com/google/domain/registry google/registry

This change renames directories in preparation for the great package
rename. The repository is now in a broken state because the code
itself hasn't been updated. However this should ensure that git
correctly preserves history for each file.
This commit is contained in:
Justine Tunney 2016-05-13 18:55:08 -04:00
parent a41677aea1
commit 5012893c1d
2396 changed files with 0 additions and 0 deletions

View file

@ -0,0 +1,26 @@
package(default_visibility = ["//java/com/google/domain/registry:registry_project"])
java_library(
name = "sheet",
srcs = glob(["*.java"]),
deps = [
"//java/com/google/api/client/googleapis/auth/oauth2",
"//java/com/google/common/base",
"//java/com/google/common/collect",
"//java/com/google/common/io",
"//java/com/google/common/net",
"//java/com/google/domain/registry/config",
"//java/com/google/domain/registry/model",
"//java/com/google/domain/registry/request",
"//java/com/google/domain/registry/util",
"//java/com/google/gdata",
"//java/com/google/gdata:spreadsheet",
"//third_party/java/appengine:appengine-api",
"//third_party/java/dagger",
"//third_party/java/joda_time",
"//third_party/java/jsr305_annotations",
"//third_party/java/jsr330_inject",
"//third_party/java/servlet/servlet_api",
],
)

View file

@ -0,0 +1,36 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import static com.google.common.base.Strings.emptyToNull;
import com.google.common.base.Optional;
import com.google.domain.registry.request.Parameter;
import dagger.Module;
import dagger.Provides;
import javax.servlet.http.HttpServletRequest;
/** Dagger module for the sheet package. */
@Module
public final class SheetModule {
@Provides
@Parameter("id")
static Optional<String> provideId(HttpServletRequest req) {
return Optional.fromNullable(emptyToNull(req.getParameter("id")));
}
}

View file

@ -0,0 +1,106 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import com.google.gdata.data.spreadsheet.CustomElementCollection;
import com.google.gdata.data.spreadsheet.ListEntry;
import com.google.gdata.data.spreadsheet.ListFeed;
import com.google.gdata.data.spreadsheet.SpreadsheetEntry;
import com.google.gdata.data.spreadsheet.WorksheetEntry;
import com.google.gdata.util.ServiceException;
import java.io.IOException;
import java.net.URL;
import java.util.List;
import javax.inject.Inject;
/** Generic data synchronization utility for Google Spreadsheets. */
class SheetSynchronizer {
private static final String SPREADSHEET_URL_PREFIX =
"https://spreadsheets.google.com/feeds/spreadsheets/";
@Inject SpreadsheetService spreadsheetService;
@Inject SheetSynchronizer() {}
/**
* Replace the contents of a Google Spreadsheet with {@code data}.
*
* <p>In order for this to work, you must create a spreadsheet with a header row, each containing
* the column name, without any spaces. All subsequent rows are considered data, so long as
* they're not blank. If you have a blank row in the middle of your data, you're going to have
* problems. You must also make sure that the spreadsheet has been shared with the API client
* credential email address.
*
* <p>The algorithm works by first assuming that the spreadsheet is sorted in the same way that
* {@code data} is sorted. It then iterates through the existing rows and comparing them to the
* items in {@code data}. Iteration continues until we either run out of rows, or items in
* {@code data}. If there's any rows remaining, they'll be deleted. If instead, items remain in
* data, they'll be inserted.
*
* @param spreadsheetId The ID of your spreadsheet. This can be obtained by opening the Google
* spreadsheet in your browser and copying the ID from the URL.
* @param data This should be a <i>sorted</i> list of rows containing the enterity of the
* spreadsheet. Each row is a map, where the key must be exactly the same as the column header
* cell in the spreadsheet, and value is an arbitrary object which will be converted to a
* string before storing it in the spreadsheet.
* @throws IOException error communicating with the GData service.
* @throws ServiceException if a system error occurred when retrieving the entry.
* @throws com.google.gdata.util.ParseException error parsing the returned entry.
* @throws com.google.gdata.util.ResourceNotFoundException if an entry URL is not valid.
* @throws com.google.gdata.util.ServiceForbiddenException if the GData service cannot get the
* entry resource due to access constraints.
* @see "https://developers.google.com/google-apps/spreadsheets/"
*/
void synchronize(String spreadsheetId, ImmutableList<ImmutableMap<String, String>> data)
throws IOException, ServiceException {
URL url = new URL(SPREADSHEET_URL_PREFIX + spreadsheetId);
SpreadsheetEntry spreadsheet = spreadsheetService.getEntry(url, SpreadsheetEntry.class);
WorksheetEntry worksheet = spreadsheet.getWorksheets().get(0);
worksheet.setRowCount(data.size());
worksheet = worksheet.update();
ListFeed listFeed = spreadsheetService.getFeed(worksheet.getListFeedUrl(), ListFeed.class);
List<ListEntry> entries = listFeed.getEntries();
int commonSize = Math.min(entries.size(), data.size());
for (int i = 0; i < commonSize; i++) {
ListEntry entry = entries.get(i);
CustomElementCollection elements = entry.getCustomElements();
boolean mutated = false;
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
if (!cell.getValue().equals(elements.getValue(cell.getKey()))) {
mutated = true;
elements.setValueLocal(cell.getKey(), cell.getValue());
}
}
if (mutated) {
entry.update();
}
}
if (data.size() > entries.size()) {
for (int i = entries.size(); i < data.size(); i++) {
ListEntry entry = listFeed.createEntry();
CustomElementCollection elements = entry.getCustomElements();
for (ImmutableMap.Entry<String, String> cell : data.get(i).entrySet()) {
elements.setValueLocal(cell.getKey(), cell.getValue());
}
listFeed.insert(entry);
}
}
}
}

View file

@ -0,0 +1,39 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.common.collect.ImmutableList;
import com.google.gdata.client.spreadsheet.SpreadsheetService;
import dagger.Module;
import dagger.Provides;
/** Dagger module for {@link SpreadsheetService}. */
@Module
public final class SpreadsheetServiceModule {
private static final String APPLICATION_NAME = "google-registry-v1";
private static final ImmutableList<String> SCOPES = ImmutableList.of(
"https://spreadsheets.google.com/feeds",
"https://docs.google.com/feeds");
@Provides
static SpreadsheetService provideSpreadsheetService(GoogleCredential credential) {
SpreadsheetService service = new SpreadsheetService(APPLICATION_NAME);
service.setOAuth2Credentials(credential.createScoped(SCOPES));
return service;
}
}

View file

@ -0,0 +1,211 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import static com.google.common.base.MoreObjects.firstNonNull;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.ABUSE;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.ADMIN;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.BILLING;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.LEGAL;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.MARKETING;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.TECH;
import static com.google.domain.registry.model.registrar.RegistrarContact.Type.WHOIS;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Ordering;
import com.google.domain.registry.model.registrar.Registrar;
import com.google.domain.registry.model.registrar.RegistrarAddress;
import com.google.domain.registry.model.registrar.RegistrarContact;
import com.google.domain.registry.util.Clock;
import com.google.domain.registry.util.DateTimeUtils;
import com.google.gdata.util.ServiceException;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import java.io.IOException;
import javax.annotation.Nullable;
import javax.inject.Inject;
/**
* Class for synchronizing all {@link Registrar} datastore objects to a Google Spreadsheet.
*
* @see SyncRegistrarsSheetAction
*/
class SyncRegistrarsSheet {
@Inject Clock clock;
@Inject SheetSynchronizer sheetSynchronizer;
@Inject SyncRegistrarsSheet() {}
/** Returns true if a {@link Registrar} entity was modified in past {@code duration}. */
boolean wasRegistrarsModifiedInLast(Duration duration) {
DateTime watermark = clock.nowUtc().minus(duration);
for (Registrar registrar : Registrar.loadAll()) {
if (DateTimeUtils.isAtOrAfter(registrar.getLastUpdateTime(), watermark)) {
return true;
}
}
return false;
}
/** Performs the synchronization operation. */
void run(String spreadsheetId) throws IOException, ServiceException {
sheetSynchronizer.synchronize(
spreadsheetId,
FluentIterable
.from(
new Ordering<Registrar>() {
@Override
public int compare(Registrar left, Registrar right) {
return left.getClientIdentifier().compareTo(right.getClientIdentifier());
}
}.immutableSortedCopy(Registrar.loadAll()))
.filter(
new Predicate<Registrar>() {
@Override
public boolean apply(Registrar registrar) {
return registrar.getType() == Registrar.Type.REAL
|| registrar.getType() == Registrar.Type.OTE;
}
})
.transform(
new Function<Registrar, ImmutableMap<String, String>>() {
@Override
public ImmutableMap<String, String> apply(Registrar registrar) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
ImmutableSortedSet<RegistrarContact> contacts = registrar.getContacts();
RegistrarAddress address =
firstNonNull(
registrar.getLocalizedAddress(),
firstNonNull(
registrar.getInternationalizedAddress(),
new RegistrarAddress.Builder()
.setStreet(ImmutableList.of("UNKNOWN"))
.setCity("UNKNOWN")
.setCountryCode("US")
.build()));
//
// °° WARNING WARNING WARNING
//
// Do not change these mappings simply because the Registrar model changed. Only
// change these mappings if the people who use the spreadsheet requested it be
// changed.
//
// These values are hard-coded because they correspond to actual spreadsheet
// columns. If you change this dictionary, then you'll need to manually add new
// columns to the registrar spreadsheets for all environments before deployment,
// and you'll need to remove deleted columns probably like a week after
// deployment.
//
builder.put("clientIdentifier", convert(registrar.getClientIdentifier()));
builder.put("registrarName", convert(registrar.getRegistrarName()));
builder.put("state", convert(registrar.getState()));
builder.put("ianaIdentifier", convert(registrar.getIanaIdentifier()));
builder.put("billingIdentifier", convert(registrar.getBillingIdentifier()));
builder.put("primaryContacts", convertContacts(contacts, byType(ADMIN)));
builder.put("techContacts", convertContacts(contacts, byType(TECH)));
builder.put("marketingContacts", convertContacts(contacts, byType(MARKETING)));
builder.put("abuseContacts", convertContacts(contacts, byType(ABUSE)));
builder.put("whoisInquiryContacts", convertContacts(contacts, byType(WHOIS)));
builder.put("legalContacts", convertContacts(contacts, byType(LEGAL)));
builder.put("billingContacts", convertContacts(contacts, byType(BILLING)));
builder.put(
"contactsMarkedAsWhoisAdmin",
convertContacts(
contacts,
new Predicate<RegistrarContact>() {
@Override
public boolean apply(RegistrarContact contact) {
return contact.getVisibleInWhoisAsAdmin();
}
}));
builder.put(
"contactsMarkedAsWhoisTech",
convertContacts(
contacts,
new Predicate<RegistrarContact>() {
@Override
public boolean apply(RegistrarContact contact) {
return contact.getVisibleInWhoisAsTech();
}
}));
builder.put("emailAddress", convert(registrar.getEmailAddress()));
builder.put("address.street", convert(address.getStreet()));
builder.put("address.city", convert(address.getCity()));
builder.put("address.state", convert(address.getState()));
builder.put("address.zip", convert(address.getZip()));
builder.put("address.countryCode", convert(address.getCountryCode()));
builder.put("phoneNumber", convert(registrar.getPhoneNumber()));
builder.put("faxNumber", convert(registrar.getFaxNumber()));
builder.put("creationTime", convert(registrar.getCreationTime()));
builder.put("lastUpdateTime", convert(registrar.getLastUpdateTime()));
builder.put("allowedTlds", convert(registrar.getAllowedTlds()));
builder.put("whoisServer", convert(registrar.getWhoisServer()));
builder.put("blockPremiumNames", convert(registrar.getBlockPremiumNames()));
builder.put("ipAddressWhitelist", convert(registrar.getIpAddressWhitelist()));
builder.put("url", convert(registrar.getUrl()));
builder.put("referralUrl", convert(registrar.getReferralUrl()));
builder.put("icannReferralEmail", convert(registrar.getIcannReferralEmail()));
return builder.build();
}
})
.toList());
}
private static String convertContacts(
Iterable<RegistrarContact> contacts, Predicate<RegistrarContact> filter) {
StringBuilder result = new StringBuilder();
boolean first = true;
for (RegistrarContact contact : contacts) {
if (!filter.apply(contact)) {
continue;
}
if (first) {
first = false;
} else {
result.append("\n");
}
result.append(contact.toStringMultilinePlainText());
}
return result.toString();
}
private static Predicate<RegistrarContact> byType(final RegistrarContact.Type type) {
return new Predicate<RegistrarContact>() {
@Override
public boolean apply(RegistrarContact contact) {
return contact.getTypes().contains(type);
}};
}
/** Converts a value to a string representation that can be stored in a spreadsheet cell. */
private static String convert(@Nullable Object value) {
if (value == null) {
return "";
} else if (value instanceof Iterable) {
return Joiner.on('\n').join((Iterable<?>) value);
} else {
return value.toString();
}
}
}

View file

@ -0,0 +1,162 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.domain.registry.export.sheet;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
import static com.google.domain.registry.request.Action.Method.POST;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.appengine.api.modules.ModulesService;
import com.google.appengine.api.modules.ModulesServiceFactory;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TaskOptions.Method;
import com.google.common.base.Optional;
import com.google.domain.registry.config.ConfigModule.Config;
import com.google.domain.registry.model.server.Lock;
import com.google.domain.registry.request.Action;
import com.google.domain.registry.request.Parameter;
import com.google.domain.registry.request.Response;
import com.google.domain.registry.util.FormattingLogger;
import com.google.domain.registry.util.NonFinalForTesting;
import com.google.gdata.util.ServiceException;
import org.joda.time.Duration;
import java.io.IOException;
import java.util.concurrent.Callable;
import javax.annotation.Nullable;
import javax.inject.Inject;
/**
* Action for synchronizing the registrars spreadsheet.
*
* <p>You can specify the spreadsheet ID by passing the "id" parameter. If this parameter is not
* specified, then the spreadsheet ID will be obtained from the registry configuration.
*
* <p>Cron will run this action hourly. So in order to minimize Google Spreadsheets I/O, this action
* will iterate through all registrars and check if any entries were modified in the past hour. If
* no modifications were made, the action will exit without performing any syncing.
*
* <p><b>Note:</b> Setting the "id" parameter will disable the registrar update check.
*
* <p>Before using this service, you should make sure all the column headers listed in this source
* file are present. You also need to share the spreadsheet with the email address from the JSON
* credential file and give it edit permission.
*
* @see SyncRegistrarsSheet
*/
@Action(path = SyncRegistrarsSheetAction.PATH, method = POST)
public class SyncRegistrarsSheetAction implements Runnable {
private enum Result {
OK(SC_OK, "Sheet successfully updated."),
NOTMODIFIED(SC_OK, "Registrars table hasn't been modified in past hour."),
LOCKED(SC_NO_CONTENT, "Another task is currently writing to this sheet; dropping task."),
MISSINGNO(SC_BAD_REQUEST, "No sheet ID specified or configured; dropping task.") {
@Override
protected void log(Exception cause) {
logger.warningfmt(cause, "%s", message);
}},
FAILED(SC_INTERNAL_SERVER_ERROR, "Spreadsheet synchronization failed") {
@Override
protected void log(Exception cause) {
logger.severefmt(cause, "%s", message);
}};
private final int statusCode;
protected final String message;
private Result(int statusCode, String message) {
this.statusCode = statusCode;
this.message = message;
}
/** Log an error message. Results that use log levels other than info should override this. */
protected void log(@Nullable Exception cause) {
logger.infofmt(cause, "%s", message);
}
private void send(Response response, @Nullable Exception cause) {
log(cause);
response.setStatus(statusCode);
response.setContentType(PLAIN_TEXT_UTF_8);
response.setPayload(String.format("%s %s\n", name(), message));
}
}
public static final String PATH = "/_dr/task/syncRegistrarsSheet";
private static final String QUEUE = "sheet";
private static final String LOCK_NAME = "Synchronize registrars sheet";
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
@NonFinalForTesting
private static ModulesService modulesService = ModulesServiceFactory.getModulesService();
@Inject Response response;
@Inject SyncRegistrarsSheet syncRegistrarsSheet;
@Inject @Config("sheetLockTimeout") Duration timeout;
@Inject @Config("sheetRegistrarId") Optional<String> idConfig;
@Inject @Config("sheetRegistrarInterval") Duration interval;
@Inject @Parameter("id") Optional<String> idParam;
@Inject SyncRegistrarsSheetAction() {}
@Override
public void run() {
final Optional<String> sheetId = idParam.or(idConfig);
if (!sheetId.isPresent()) {
Result.MISSINGNO.send(response, null);
return;
}
if (!idParam.isPresent()) {
// TODO(b/19082368): Use a cursor.
if (!syncRegistrarsSheet.wasRegistrarsModifiedInLast(interval)) {
Result.NOTMODIFIED.send(response, null);
return;
}
}
String sheetLockName = String.format("%s: %s", LOCK_NAME, sheetId.get());
Callable<Void> runner = new Callable<Void>() {
@Nullable
@Override
public Void call() throws IOException {
try {
syncRegistrarsSheet.run(sheetId.get());
Result.OK.send(response, null);
} catch (IOException | ServiceException e) {
Result.FAILED.send(response, e);
}
return null;
}
};
if (!Lock.executeWithLocks(runner, getClass(), "", timeout, sheetLockName)) {
// If we fail to acquire the lock, it probably means lots of updates are happening at once, in
// which case it should be safe to not bother. The task queue definition should *not* specify
// max-concurrent-requests for this very reason.
Result.LOCKED.send(response, null);
}
}
/** Creates, enqueues, and returns a new backend task to sync registrar spreadsheets. */
public static TaskHandle enqueueBackendTask() {
String hostname = modulesService.getVersionHostname("backend", null);
return getQueue(QUEUE).add(withUrl(PATH).method(Method.GET).header("Host", hostname));
}
}

View file

@ -0,0 +1,16 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
@javax.annotation.ParametersAreNonnullByDefault
package com.google.domain.registry.export.sheet;