diff --git a/java/google/registry/config/RegistryEnvironment.java b/java/google/registry/config/RegistryEnvironment.java index 0c194ca73..16aec6da6 100644 --- a/java/google/registry/config/RegistryEnvironment.java +++ b/java/google/registry/config/RegistryEnvironment.java @@ -15,6 +15,7 @@ package google.registry.config; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; import javax.annotation.Nullable; /** Registry environments. */ @@ -50,7 +51,7 @@ public enum RegistryEnvironment { /** Returns environment configured by system property {@value #PROPERTY}. */ public static RegistryEnvironment get() { - return valueOf(System.getProperty(PROPERTY, UNITTEST.name()).toUpperCase()); + return valueOf(Ascii.toUpperCase(System.getProperty(PROPERTY, UNITTEST.name()))); } /** diff --git a/java/google/registry/export/DatastoreBackupInfo.java b/java/google/registry/export/DatastoreBackupInfo.java index 312d4b220..a9a647827 100644 --- a/java/google/registry/export/DatastoreBackupInfo.java +++ b/java/google/registry/export/DatastoreBackupInfo.java @@ -21,6 +21,7 @@ import static org.joda.time.DateTimeZone.UTC; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.Text; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; @@ -140,7 +141,7 @@ public class DatastoreBackupInfo { "Status: " + getStatus(), "Started: " + startTime, "Ended: " + completeTime.orNull(), - "Duration: " + getRunningTime().toPeriod().toString().substring(2).toLowerCase(), + "Duration: " + Ascii.toLowerCase(getRunningTime().toPeriod().toString().substring(2)), "GCS: " + gcsFilename.orNull(), "Kinds: " + kinds, ""); diff --git a/java/google/registry/model/domain/fee/FeeCommandDescriptor.java b/java/google/registry/model/domain/fee/FeeCommandDescriptor.java index cc61f9944..d410cbf0c 100644 --- a/java/google/registry/model/domain/fee/FeeCommandDescriptor.java +++ b/java/google/registry/model/domain/fee/FeeCommandDescriptor.java @@ -14,6 +14,7 @@ package google.registry.model.domain.fee; +import com.google.common.base.Ascii; import com.google.common.base.CharMatcher; import google.registry.model.ImmutableObject; import javax.xml.bind.annotation.XmlAttribute; @@ -56,7 +57,7 @@ public class FeeCommandDescriptor extends ImmutableObject { // Require the xml string to be lowercase. if (command != null && CharMatcher.javaLowerCase().matchesAllOf(command)) { try { - return CommandName.valueOf(command.toUpperCase()); + return CommandName.valueOf(Ascii.toUpperCase(command)); } catch (IllegalArgumentException e) { // Swallow this and return UNKNOWN below because there's no matching CommandName. } diff --git a/java/google/registry/model/domain/launch/LaunchNotice.java b/java/google/registry/model/domain/launch/LaunchNotice.java index 92986d9cf..4858d14f9 100644 --- a/java/google/registry/model/domain/launch/LaunchNotice.java +++ b/java/google/registry/model/domain/launch/LaunchNotice.java @@ -20,6 +20,7 @@ import static com.google.common.io.BaseEncoding.base16; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import com.google.common.base.Ascii; import com.google.common.base.CharMatcher; import com.google.common.base.Optional; import com.google.common.primitives.Ints; @@ -101,7 +102,7 @@ public class LaunchNotice extends ImmutableObject { String tcnId = getNoticeId().getTcnId(); checkArgument(tcnId.length() == 27); - int checksum = Ints.fromByteArray(base16().decode(tcnId.substring(0, 8).toUpperCase())); + int checksum = Ints.fromByteArray(base16().decode(Ascii.toUpperCase(tcnId.substring(0, 8)))); String noticeId = tcnId.substring(8); checkArgument(CharMatcher.inRange('0', '9').matchesAllOf(noticeId)); diff --git a/java/google/registry/rdap/RdapJsonFormatter.java b/java/google/registry/rdap/RdapJsonFormatter.java index d439b8d02..263fb1c07 100644 --- a/java/google/registry/rdap/RdapJsonFormatter.java +++ b/java/google/registry/rdap/RdapJsonFormatter.java @@ -85,8 +85,9 @@ public class RdapJsonFormatter { static final String NOTICES = "notices"; private static final String REMARKS = "remarks"; - /** Status values specified in RFC 7483 § 10.2.2. */ private enum RdapStatus { + + // Status values specified in RFC 7483 § 10.2.2. VALIDATED("validated"), RENEW_PROHIBITED("renew prohibited"), UPDATE_PROHIBITED("update prohibited"), @@ -104,7 +105,26 @@ public class RdapJsonFormatter { PENDING_RENEW("pending renew"), PENDING_TRANSFER("pending transfer"), PENDING_UPDATE("pending update"), - PENDING_DELETE("pending delete"); + PENDING_DELETE("pending delete"), + + // Additional status values defined in + // https://tools.ietf.org/html/draft-ietf-regext-epp-rdap-status-mapping-01. + ADD_PERIOD("add period"), + AUTO_RENEW_PERIOD("auto renew period"), + CLIENT_DELETE_PROHIBITED("client delete prohibited"), + CLIENT_HOLD("client hold"), + CLIENT_RENEW_PROHIBITED("client renew prohibited"), + CLIENT_TRANSFER_PROHIBITED("client transfer prohibited"), + CLIENT_UPDATE_PROHIBITED("client update prohibited"), + PENDING_RESTORE("pending restore"), + REDEMPTION_PERIOD("redemption period"), + RENEW_PERIOD("renew period"), + SERVER_DELETE_PROHIBITED("server deleted prohibited"), + SERVER_RENEW_PROHIBITED("server renew prohibited"), + SERVER_TRANSFER_PROHIBITED("server transfer prohibited"), + SERVER_UPDATE_PROHIBITED("server update prohibited"), + SERVER_HOLD("server hold"), + TRANSFER_PERIOD("transfer period"); /** Value as it appears in RDAP messages. */ private final String rfc7483String; @@ -123,23 +143,30 @@ public class RdapJsonFormatter { private static final ImmutableMap statusToRdapStatusMap = Maps.immutableEnumMap( new ImmutableMap.Builder() - .put(StatusValue.CLIENT_DELETE_PROHIBITED, RdapStatus.DELETE_PROHIBITED) - .put(StatusValue.CLIENT_HOLD, RdapStatus.INACTIVE) - .put(StatusValue.CLIENT_RENEW_PROHIBITED, RdapStatus.RENEW_PROHIBITED) - .put(StatusValue.CLIENT_TRANSFER_PROHIBITED, RdapStatus.TRANSFER_PROHIBITED) - .put(StatusValue.CLIENT_UPDATE_PROHIBITED, RdapStatus.UPDATE_PROHIBITED) + // StatusValue.ADD_PERIOD not defined in our system + // StatusValue.AUTO_RENEW_PERIOD not defined in our system + .put(StatusValue.CLIENT_DELETE_PROHIBITED, RdapStatus.CLIENT_DELETE_PROHIBITED) + .put(StatusValue.CLIENT_HOLD, RdapStatus.CLIENT_HOLD) + .put(StatusValue.CLIENT_RENEW_PROHIBITED, RdapStatus.CLIENT_RENEW_PROHIBITED) + .put(StatusValue.CLIENT_TRANSFER_PROHIBITED, RdapStatus.CLIENT_TRANSFER_PROHIBITED) + .put(StatusValue.CLIENT_UPDATE_PROHIBITED, RdapStatus.CLIENT_UPDATE_PROHIBITED) .put(StatusValue.INACTIVE, RdapStatus.INACTIVE) .put(StatusValue.LINKED, RdapStatus.ASSOCIATED) .put(StatusValue.OK, RdapStatus.ACTIVE) .put(StatusValue.PENDING_CREATE, RdapStatus.PENDING_CREATE) .put(StatusValue.PENDING_DELETE, RdapStatus.PENDING_DELETE) + // StatusValue.PENDING_RENEW not defined in our system + // StatusValue.PENDING_RESTORE not defined in our system .put(StatusValue.PENDING_TRANSFER, RdapStatus.PENDING_TRANSFER) .put(StatusValue.PENDING_UPDATE, RdapStatus.PENDING_UPDATE) - .put(StatusValue.SERVER_DELETE_PROHIBITED, RdapStatus.DELETE_PROHIBITED) - .put(StatusValue.SERVER_HOLD, RdapStatus.INACTIVE) - .put(StatusValue.SERVER_RENEW_PROHIBITED, RdapStatus.RENEW_PROHIBITED) - .put(StatusValue.SERVER_TRANSFER_PROHIBITED, RdapStatus.TRANSFER_PROHIBITED) - .put(StatusValue.SERVER_UPDATE_PROHIBITED, RdapStatus.UPDATE_PROHIBITED) + // StatusValue.REDEMPTION_PERIOD not defined in our system + // StatusValue.RENEW_PERIOD not defined in our system + .put(StatusValue.SERVER_DELETE_PROHIBITED, RdapStatus.SERVER_DELETE_PROHIBITED) + .put(StatusValue.SERVER_HOLD, RdapStatus.SERVER_HOLD) + .put(StatusValue.SERVER_RENEW_PROHIBITED, RdapStatus.SERVER_RENEW_PROHIBITED) + .put(StatusValue.SERVER_TRANSFER_PROHIBITED, RdapStatus.SERVER_TRANSFER_PROHIBITED) + .put(StatusValue.SERVER_UPDATE_PROHIBITED, RdapStatus.SERVER_UPDATE_PROHIBITED) + // StatusValue.TRANSFER_PERIOD not defined in our system .build()); /** Role values specified in RFC 7483 § 10.2.4. */ @@ -189,7 +216,7 @@ public class RdapJsonFormatter { } } - /** Map of EPP status values to the RDAP equivalents. */ + /** Map of EPP event values to the RDAP equivalents. */ private static final ImmutableMap historyEntryTypeToRdapEventActionMap = Maps.immutableEnumMap( diff --git a/java/google/registry/rde/DomainResourceToXjcConverter.java b/java/google/registry/rde/DomainResourceToXjcConverter.java index 921decf08..66f0bcc27 100644 --- a/java/google/registry/rde/DomainResourceToXjcConverter.java +++ b/java/google/registry/rde/DomainResourceToXjcConverter.java @@ -14,6 +14,7 @@ package google.registry.rde; +import com.google.common.base.Ascii; import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Ref; @@ -280,7 +281,7 @@ final class DomainResourceToXjcConverter { private static XjcDomainContactType convertDesignatedContact(DesignatedContact model) { XjcDomainContactType bean = new XjcDomainContactType(); ContactResource contact = model.getContactRef().get(); - bean.setType(XjcDomainContactAttrType.fromValue(model.getType().toString().toLowerCase())); + bean.setType(XjcDomainContactAttrType.fromValue(Ascii.toLowerCase(model.getType().toString()))); bean.setValue(contact.getContactId()); return bean; } diff --git a/java/google/registry/rde/RdeImportUtils.java b/java/google/registry/rde/RdeImportUtils.java new file mode 100644 index 000000000..f3ce5dc23 --- /dev/null +++ b/java/google/registry/rde/RdeImportUtils.java @@ -0,0 +1,101 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.rde; + +import static com.google.common.base.Preconditions.checkState; + +import com.googlecode.objectify.Key; +import com.googlecode.objectify.Work; +import google.registry.model.contact.ContactResource; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.index.ForeignKeyIndex; +import google.registry.model.ofy.Ofy; +import google.registry.util.Clock; +import google.registry.util.FormattingLogger; +import javax.inject.Inject; + +/** Utility functions for escrow file import. */ +public final class RdeImportUtils { + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + private final Ofy ofy; + private final Clock clock; + + @Inject + public RdeImportUtils(Ofy ofy, Clock clock) { + this.ofy = ofy; + this.clock = clock; + } + + /** + * Imports a contact from an escrow file. + * + *

The contact will only be imported if it has not been previously imported. + * + *

If the contact is imported, {@link ForeignKeyIndex} and {@link EppResourceIndex} are also + * created. + * + * @return true if the contact was created or updated, false otherwise. + */ + public boolean importContact(final ContactResource resource) { + return ofy.transact( + new Work() { + @Override + public Boolean run() { + ContactResource existing = ofy.load().key(Key.create(resource)).now(); + if (existing == null) { + ForeignKeyIndex existingForeignKeyIndex = + ForeignKeyIndex.load( + ContactResource.class, resource.getContactId(), clock.nowUtc()); + // foreign key index should not exist, since existing contact was not found. + checkState( + existingForeignKeyIndex == null, + String.format( + "New contact resource has existing foreign key index. " + + "contactId=%s, repoId=%s", + resource.getContactId(), resource.getRepoId())); + ofy.save().entity(resource); + ofy.save().entity(ForeignKeyIndex.create(resource, resource.getDeletionTime())); + ofy.save().entity(EppResourceIndex.create(Key.create(resource))); + logger.infofmt( + "Imported contact resource - ROID=%s, id=%s", + resource.getRepoId(), resource.getContactId()); + return true; + } else if (!existing.getRepoId().equals(resource.getRepoId())) { + logger.warningfmt( + "Existing contact with same contact id but different ROID. " + + "contactId=%s, existing ROID=%s, new ROID=%s", + resource.getContactId(), existing.getRepoId(), resource.getRepoId()); + } + return false; + } + }); + } +} diff --git a/java/google/registry/tmch/LordnLog.java b/java/google/registry/tmch/LordnLog.java index ca8f1c82a..3543c6a9d 100644 --- a/java/google/registry/tmch/LordnLog.java +++ b/java/google/registry/tmch/LordnLog.java @@ -18,6 +18,7 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.Ascii; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.re2j.Pattern; @@ -218,7 +219,7 @@ public final class LordnLog implements Iterable> // + , whether the LORDN file has been accepted for // processing by the TMDB. Possible values are "accepted" or // "rejected". - Status status = Status.valueOf(firstLine.get(4).toUpperCase()); + Status status = Status.valueOf(Ascii.toUpperCase(firstLine.get(4))); // + , whether the LORDN Log has any warning result // codes. Possible values are "no-warnings" or "warnings- @@ -229,8 +230,11 @@ public final class LordnLog implements Iterable> // processed in the LORDN file. int dnLines = Integer.parseInt(firstLine.get(6)); int actual = lines.size() - 2; - checkArgument(dnLines == actual, - "Line 1: Number of entries (%d) differs from declaration (%d)", actual, dnLines); + checkArgument( + dnLines == actual, + "Line 1: Number of entries (%s) differs from declaration (%s)", + String.valueOf(actual), + String.valueOf(dnLines)); // Second line contains headers: roid,result-code checkArgument(lines.get(1).equals("roid,result-code"), @@ -244,8 +248,7 @@ public final class LordnLog implements Iterable> "Line %d: Expected 2 elements, found %d", i + 1, currentLine.size())); String roid = currentLine.get(0); int code = Integer.parseInt(currentLine.get(1)); - Result result = checkNotNull(RESULTS.get(code), - "Line %d: Unknown result code: %d", i, code); + Result result = checkNotNull(RESULTS.get(code), "Line %s: Unknown result code: %s", i, code); builder.put(roid, result); } diff --git a/java/google/registry/tools/AllocateDomainCommand.java b/java/google/registry/tools/AllocateDomainCommand.java index 64ff8ea78..40dc3f34a 100644 --- a/java/google/registry/tools/AllocateDomainCommand.java +++ b/java/google/registry/tools/AllocateDomainCommand.java @@ -26,6 +26,7 @@ import static google.registry.tools.CommandUtilities.addHeader; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Splitter; @@ -134,7 +135,7 @@ final class AllocateDomainCommand extends MutatingEppToolCommand { ImmutableMap.Builder contactsMapBuilder = new ImmutableMap.Builder<>(); for (DesignatedContact contact : application.getContacts()) { contactsMapBuilder.put( - contact.getType().toString().toLowerCase(), + Ascii.toLowerCase(contact.getType().toString()), contact.getContactRef().get().getForeignKey()); } LaunchNotice launchNotice = application.getLaunchNotice(); diff --git a/java/google/registry/tools/CommandUtilities.java b/java/google/registry/tools/CommandUtilities.java index 95cd72689..553a3dbbd 100644 --- a/java/google/registry/tools/CommandUtilities.java +++ b/java/google/registry/tools/CommandUtilities.java @@ -14,6 +14,7 @@ package google.registry.tools; +import com.google.common.base.Ascii; import com.google.common.base.Strings; /** Container class for static utility methods. */ @@ -25,6 +26,6 @@ class CommandUtilities { /** Prompts for yes/no input using promptText, defaulting to no. */ static boolean promptForYes(String promptText) { - return System.console().readLine(promptText + " (y/N): ").toUpperCase().startsWith("Y"); + return Ascii.toUpperCase(System.console().readLine(promptText + " (y/N): ")).startsWith("Y"); } } diff --git a/java/google/registry/tools/ConvertIdnCommand.java b/java/google/registry/tools/ConvertIdnCommand.java index 10a45b910..c85411e57 100644 --- a/java/google/registry/tools/ConvertIdnCommand.java +++ b/java/google/registry/tools/ConvertIdnCommand.java @@ -19,6 +19,7 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.google.common.base.Ascii; import google.registry.tools.Command.GtechCommand; import google.registry.util.Idn; import java.io.IOException; @@ -37,7 +38,7 @@ final class ConvertIdnCommand implements Command, GtechCommand { public void run() throws IOException { for (String label : mainParameters) { if (label.startsWith(ACE_PREFIX)) { - System.out.println(Idn.toUnicode(label.toLowerCase())); + System.out.println(Idn.toUnicode(Ascii.toLowerCase(label))); } else { System.out.println(canonicalizeDomainName(label)); } diff --git a/java/google/registry/tools/DomainApplicationInfoCommand.java b/java/google/registry/tools/DomainApplicationInfoCommand.java index 70ecaee6a..891dcb804 100644 --- a/java/google/registry/tools/DomainApplicationInfoCommand.java +++ b/java/google/registry/tools/DomainApplicationInfoCommand.java @@ -18,6 +18,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.google.common.base.Ascii; import com.google.template.soy.data.SoyMapData; import google.registry.model.domain.launch.LaunchPhase; import google.registry.tools.Command.GtechCommand; @@ -53,8 +54,8 @@ final class DomainApplicationInfoCommand extends EppToolCommand implements Gtech @Override void initEppToolCommand() { - LaunchPhase launchPhase = - checkArgumentNotNull(LaunchPhase.fromValue(phase.toLowerCase()), "Illegal launch phase."); + LaunchPhase launchPhase = checkArgumentNotNull( + LaunchPhase.fromValue(Ascii.toLowerCase(phase)), "Illegal launch phase."); setSoyTemplate( DomainApplicationInfoSoyInfo.getInstance(), diff --git a/java/google/registry/tools/RegistryToolEnvironment.java b/java/google/registry/tools/RegistryToolEnvironment.java index 7567cbfcf..85a7ae2e7 100644 --- a/java/google/registry/tools/RegistryToolEnvironment.java +++ b/java/google/registry/tools/RegistryToolEnvironment.java @@ -17,6 +17,7 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import google.registry.config.RegistryEnvironment; @@ -61,7 +62,7 @@ enum RegistryToolEnvironment { * @see #get() */ static RegistryToolEnvironment parseFromArgs(String[] args) { - return valueOf(getFlagValue(args, FLAGS).toUpperCase()); + return valueOf(Ascii.toUpperCase(getFlagValue(args, FLAGS))); } /** diff --git a/java/google/registry/tools/server/VerifyOteAction.java b/java/google/registry/tools/server/VerifyOteAction.java index 5c8b63ad4..d6f990062 100644 --- a/java/google/registry/tools/server/VerifyOteAction.java +++ b/java/google/registry/tools/server/VerifyOteAction.java @@ -23,6 +23,7 @@ import static google.registry.util.CollectionUtils.isNullOrEmpty; import static google.registry.util.DomainNameUtils.ACE_PREFIX; import static java.util.Arrays.asList; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Predicate; @@ -250,7 +251,7 @@ public class VerifyOteAction implements Runnable, JsonAction { /** Returns a more human-readable translation of the enum constant. */ String description() { - return this.name().replace('_', ' ').toLowerCase(); + return Ascii.toLowerCase(this.name().replace('_', ' ')); } /** An {@link EppInput} might match multiple actions, so check if this action matches. */ diff --git a/java/google/registry/ui/forms/FormField.java b/java/google/registry/ui/forms/FormField.java index 0dc499b04..b8181a8f5 100644 --- a/java/google/registry/ui/forms/FormField.java +++ b/java/google/registry/ui/forms/FormField.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import com.google.common.base.Ascii; import com.google.common.base.CharMatcher; import com.google.common.base.Function; import com.google.common.base.Functions; @@ -30,6 +31,7 @@ import com.google.common.collect.Range; import com.google.re2j.Pattern; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import javax.annotation.Detainted; @@ -560,7 +562,7 @@ public final class FormField { @Nullable @Override public String apply(@Nullable String input) { - return input != null ? input.toUpperCase() : null; + return input != null ? input.toUpperCase(Locale.ENGLISH) : null; }}; private static final Function LOWERCASE_FUNCTION = @@ -568,7 +570,7 @@ public final class FormField { @Nullable @Override public String apply(@Nullable String input) { - return input != null ? input.toLowerCase() : null; + return input != null ? input.toLowerCase(Locale.ENGLISH) : null; }}; private static final Function REQUIRED_FUNCTION = @@ -587,8 +589,8 @@ public final class FormField { @Nullable @Override public Object apply(@Nullable Object input) { - return input instanceof CharSequence && ((CharSequence) input).length() == 0 - || input instanceof Collection && ((Collection) input).isEmpty() + return ((input instanceof CharSequence) && (((CharSequence) input).length() == 0)) + || ((input instanceof Collection) && ((Collection) input).isEmpty()) ? null : input; }}; @@ -709,7 +711,7 @@ public final class FormField { @Override public C apply(@Nullable O input) { try { - return input != null ? Enum.valueOf(enumClass, ((String) input).toUpperCase()) : null; + return input != null ? Enum.valueOf(enumClass, Ascii.toUpperCase((String) input)) : null; } catch (IllegalArgumentException e) { throw new FormFieldException( String.format("Enum %s does not contain '%s'", enumClass.getSimpleName(), input)); diff --git a/java/google/registry/ui/server/RegistrarFormFields.java b/java/google/registry/ui/server/RegistrarFormFields.java index b7f9f3761..8150ee387 100644 --- a/java/google/registry/ui/server/RegistrarFormFields.java +++ b/java/google/registry/ui/server/RegistrarFormFields.java @@ -19,6 +19,7 @@ import static com.google.common.collect.Range.atMost; import static com.google.common.collect.Range.closed; import static google.registry.util.DomainNameUtils.canonicalizeDomainName; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -354,7 +355,7 @@ public final class RegistrarFormFields { } for (String state : stateField.extractUntyped(args).asSet()) { if ("US".equals(countryCode)) { - state = state.toUpperCase(); + state = Ascii.toUpperCase(state); if (!StateCode.US_MAP.containsKey(state)) { throw new FormFieldException(stateField, "Unknown US state code."); } diff --git a/java/google/registry/util/Idn.java b/java/google/registry/util/Idn.java index ffb66dd20..dfc02f2ce 100644 --- a/java/google/registry/util/Idn.java +++ b/java/google/registry/util/Idn.java @@ -57,8 +57,8 @@ public final class Idn { } /** - * Translates a string from Unicode to ASCII Compatible Encoding (ACE), as defined by the ToASCII - * operation of RFC 3490. + * Translates a string from ASCII Compatible Encoding (ACE) to Unicode, as defined by the + * ToUnicode operation of RFC 3490. * *

This method always uses UTS46 transitional * processing. diff --git a/java/google/registry/util/RegistrarUtils.java b/java/google/registry/util/RegistrarUtils.java index f92901890..8f774bd40 100644 --- a/java/google/registry/util/RegistrarUtils.java +++ b/java/google/registry/util/RegistrarUtils.java @@ -16,11 +16,13 @@ package google.registry.util; import static com.google.common.base.CharMatcher.javaLetterOrDigit; +import com.google.common.base.Ascii; + /** Utilities for working with {@code Registrar} objects. */ public class RegistrarUtils { /** Strip out anything that isn't a letter or digit, and lowercase. */ public static String normalizeRegistrarName(String name) { - return javaLetterOrDigit().retainFrom(name).toLowerCase(); + return Ascii.toLowerCase(javaLetterOrDigit().retainFrom(name)); } /** @@ -29,6 +31,6 @@ public class RegistrarUtils { * in Datastore, and is suitable for use in email addresses. */ public static String normalizeClientId(String clientId) { - return clientId.toLowerCase().replaceAll("[^a-z0-9\\-]", ""); + return Ascii.toLowerCase(clientId).replaceAll("[^a-z0-9\\-]", ""); } } diff --git a/java/google/registry/util/UrlFetchUtils.java b/java/google/registry/util/UrlFetchUtils.java index 205d2fd27..30ff07108 100644 --- a/java/google/registry/util/UrlFetchUtils.java +++ b/java/google/registry/util/UrlFetchUtils.java @@ -26,6 +26,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.appengine.api.urlfetch.HTTPHeader; import com.google.appengine.api.urlfetch.HTTPRequest; import com.google.appengine.api.urlfetch.HTTPResponse; +import com.google.common.base.Ascii; import com.google.common.base.Optional; import com.google.common.net.MediaType; import java.security.NoSuchAlgorithmException; @@ -49,9 +50,9 @@ public final class UrlFetchUtils { } private static Optional getHeaderFirstInternal(Iterable hdrs, String name) { - name = name.toLowerCase(); + name = Ascii.toLowerCase(name); for (HTTPHeader header : hdrs) { - if (header.getName().toLowerCase().equals(name)) { + if (Ascii.toLowerCase(header.getName()).equals(name)) { return Optional.of(header.getValue()); } } diff --git a/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java b/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java index bc69bd281..686c04954 100644 --- a/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java +++ b/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java @@ -259,12 +259,12 @@ public class DnsUpdateWriterTest { Update update, String resourceName, int recordType, String... resourceData) { ArrayList expectedData = new ArrayList<>(); for (String resourceDatum : resourceData) { - expectedData.add(resourceDatum.toLowerCase()); + expectedData.add(resourceDatum); } ArrayList actualData = new ArrayList<>(); for (Record record : findUpdateRecords(update, resourceName, recordType)) { - actualData.add(record.rdataToString().toLowerCase()); + actualData.add(record.rdataToString()); } assertThat(actualData).containsExactlyElementsIn(expectedData); } diff --git a/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java b/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java index 3043a9988..9730ddbd4 100644 --- a/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java +++ b/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java @@ -26,6 +26,7 @@ import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.GenericEppResourceSubject.assertAboutEppResources; import static google.registry.util.DateTimeUtils.END_OF_TIME; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Ref; import google.registry.flows.Flow; @@ -113,7 +114,7 @@ public class DomainTransferFlowTestCase createTld(tld); contact = persistActiveContact("jd1234"); domain = new DomainResource.Builder() - .setRepoId("1-".concat(tld.toUpperCase())) + .setRepoId("1-".concat(Ascii.toUpperCase(tld))) .setFullyQualifiedDomainName(label + "." + tld) .setCurrentSponsorClientId("TheRegistrar") .setCreationClientId("TheRegistrar") @@ -157,7 +158,7 @@ public class DomainTransferFlowTestCase .build()); subordinateHost = persistResource( new HostResource.Builder() - .setRepoId("2-".concat(tld.toUpperCase())) + .setRepoId("2-".concat(Ascii.toUpperCase(tld))) .setFullyQualifiedHostName("ns1." + label + "." + tld) .setCurrentSponsorClientId("TheRegistrar") .setCreationClientId("TheRegistrar") diff --git a/javatests/google/registry/rdap/testdata/rdap_domain.json b/javatests/google/registry/rdap/testdata/rdap_domain.json index 8ab26c32c..c8441b57e 100644 --- a/javatests/google/registry/rdap/testdata/rdap_domain.json +++ b/javatests/google/registry/rdap/testdata/rdap_domain.json @@ -1,9 +1,9 @@ { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "handle": "%HANDLE%", "links": [ diff --git a/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json b/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json index d97e09edb..e6787e609 100644 --- a/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json +++ b/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json @@ -1,9 +1,9 @@ { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "unicodeName": "%NAME%", "handle": "%HANDLE%", diff --git a/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json b/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json index a8f24a0f9..35b480286 100644 --- a/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json +++ b/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json @@ -2,10 +2,10 @@ "domainSearchResults": [ { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "handle": "21-EXAMPLE", "links": [ @@ -361,10 +361,10 @@ }, { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "handle": "C-LOL", "links": [ diff --git a/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json b/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json index 85fb8e076..6befaae8e 100644 --- a/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json +++ b/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json @@ -5,10 +5,10 @@ "unicodeName" : "cat.みんな", "status" : [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "links" : [ diff --git a/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json b/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json index 49469ca72..fe91a0ee7 100644 --- a/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json +++ b/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json @@ -5,11 +5,11 @@ "unicodeName" : "fish.みんな", "status" : [ - "delete prohibited", + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", "inactive", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "server update prohibited" ], "links" : [ diff --git a/javatests/google/registry/rde/RdeImportUtilsTest.java b/javatests/google/registry/rde/RdeImportUtilsTest.java new file mode 100644 index 000000000..29bbc2c06 --- /dev/null +++ b/javatests/google/registry/rde/RdeImportUtilsTest.java @@ -0,0 +1,153 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.rde; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.testing.DatastoreHelper.persistResource; + +import com.google.common.base.Predicate; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; +import com.googlecode.objectify.Key; +import com.googlecode.objectify.Work; +import google.registry.model.EppResource; +import google.registry.model.contact.ContactResource; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.index.EppResourceIndexBucket; +import google.registry.model.index.ForeignKeyIndex; +import google.registry.testing.AppEngineRule; +import google.registry.testing.FakeClock; +import google.registry.testing.ShardableTestCase; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link RdeImportUtils} */ +@RunWith(JUnit4.class) +public class RdeImportUtilsTest extends ShardableTestCase { + + @Rule + public final AppEngineRule appEngine = AppEngineRule.builder() + .withDatastore() + .build(); + + private RdeImportUtils rdeImportUtils; + private FakeClock clock; + + @Before + public void before() { + clock = new FakeClock(); + clock.setTo(DateTime.now()); + rdeImportUtils = new RdeImportUtils(ofy(), clock); + } + + /** Verifies import of a contact that has not been previously imported */ + @Test + public void testImportNewContact() { + ContactResource newContact = buildNewContact(); + assertThat(rdeImportUtils.importContact(newContact)).isTrue(); + assertEppResourceIndexEntityFor(newContact); + assertForeignKeyIndexFor(newContact); + + // verify the new contact was saved + ContactResource saved = getContact("TEST-123"); + assertThat(saved).isNotNull(); + assertThat(saved.getContactId()).isEqualTo(newContact.getContactId()); + assertThat(saved.getEmailAddress()).isEqualTo(newContact.getEmailAddress()); + assertThat(saved.getLastEppUpdateTime()).isEqualTo(newContact.getLastEppUpdateTime()); + } + + /** Verifies that a contact will not be imported more than once */ + @Test + public void testImportExistingContact() { + ContactResource newContact = buildNewContact(); + persistResource(newContact); + ContactResource updatedContact = newContact.asBuilder() + .setLastEppUpdateTime(newContact.getLastEppUpdateTime().plusSeconds(1)) + .build(); + assertThat(rdeImportUtils.importContact(updatedContact)).isFalse(); + + // verify the updated contact was saved + ContactResource saved = getContact("TEST-123"); + assertThat(saved).isNotNull(); + assertThat(saved.getContactId()).isEqualTo(newContact.getContactId()); + assertThat(saved.getEmailAddress()).isEqualTo(newContact.getEmailAddress()); + assertThat(saved.getLastEppUpdateTime()).isEqualTo(newContact.getLastEppUpdateTime()); + } + + private static ContactResource buildNewContact() { + return new ContactResource.Builder() + .setContactId("sh8013") + .setEmailAddress("jdoe@example.com") + .setLastEppUpdateTime(DateTime.parse("2010-10-10T00:00:00.000Z")) + .setRepoId("TEST-123") + .build(); + } + + private static ContactResource getContact(String repoId) { + final Key key = Key.create(ContactResource.class, repoId); + return ofy().transact(new Work() { + + @Override + public ContactResource run() { + return ofy().load().key(key).now(); + }}); + } + + /** + * Confirms that a ForeignKeyIndex exists in the datastore for a given resource. + */ + private static void assertForeignKeyIndexFor(final T resource) { + assertThat(ForeignKeyIndex.load(resource.getClass(), resource.getForeignKey(), DateTime.now())) + .isNotNull(); + } + + /** + * Confirms that an EppResourceIndex entity exists in datastore for a given resource. + */ + private static void assertEppResourceIndexEntityFor(final T resource) { + ImmutableList indices = FluentIterable + .from(ofy().load() + .type(EppResourceIndex.class) + .filter("kind", Key.getKind(resource.getClass()))) + .filter(new Predicate() { + @Override + public boolean apply(EppResourceIndex index) { + return index.getReference().get().equals(resource); + }}) + .toList(); + assertThat(indices).hasSize(1); + assertThat(indices.get(0).getBucket()) + .isEqualTo(EppResourceIndexBucket.getBucketKey(Key.create(resource))); + } +} diff --git a/javatests/google/registry/rde/RdeReportActionTest.java b/javatests/google/registry/rde/RdeReportActionTest.java index 142f3768f..ac7227d61 100644 --- a/javatests/google/registry/rde/RdeReportActionTest.java +++ b/javatests/google/registry/rde/RdeReportActionTest.java @@ -38,6 +38,7 @@ import com.google.appengine.api.urlfetch.URLFetchService; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import google.registry.config.RegistryConfig; @@ -194,7 +195,7 @@ public class RdeReportActionTest { private static ImmutableMap mapifyHeaders(Iterable headers) { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); for (HTTPHeader header : headers) { - builder.put(header.getName().replace('-', '_').toUpperCase(), header.getValue()); + builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue()); } return builder.build(); } diff --git a/javatests/google/registry/testing/DatastoreHelper.java b/javatests/google/registry/testing/DatastoreHelper.java index a4ef02327..6b046c782 100644 --- a/javatests/google/registry/testing/DatastoreHelper.java +++ b/javatests/google/registry/testing/DatastoreHelper.java @@ -36,6 +36,7 @@ import static google.registry.util.ResourceUtils.readResourceUtf8; import static java.util.Arrays.asList; import static org.joda.money.CurrencyUnit.USD; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Predicate; @@ -360,7 +361,7 @@ public class DatastoreHelper { } public static void createTld(String tld, ImmutableSortedMap tldStates) { - createTld(tld, tld.replaceFirst(ACE_PREFIX_REGEX, "").toUpperCase(), tldStates); + createTld(tld, Ascii.toUpperCase(tld.replaceFirst(ACE_PREFIX_REGEX, "")), tldStates); } public static void createTld( diff --git a/javatests/google/registry/testing/TaskQueueHelper.java b/javatests/google/registry/testing/TaskQueueHelper.java index d2c786ae5..020e2987b 100644 --- a/javatests/google/registry/testing/TaskQueueHelper.java +++ b/javatests/google/registry/testing/TaskQueueHelper.java @@ -30,6 +30,7 @@ import static java.util.Arrays.asList; import com.google.appengine.api.taskqueue.dev.QueueStateInfo; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.HeaderWrapper; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Predicate; @@ -94,7 +95,7 @@ public class TaskQueueHelper { public TaskMatcher header(String name, String value) { // Lowercase for case-insensitive comparison. - expected.headers.put(name.toLowerCase(), value); + expected.headers.put(Ascii.toLowerCase(name), value); return this; } @@ -310,7 +311,7 @@ public class TaskQueueHelper { for (HeaderWrapper header : info.getHeaders()) { // Lowercase header name for comparison since HTTP // header names are case-insensitive. - headerBuilder.put(header.getKey().toLowerCase(), header.getValue()); + headerBuilder.put(Ascii.toLowerCase(header.getKey()), header.getValue()); } this.headers = headerBuilder.build(); ImmutableMultimap.Builder inputParams = new ImmutableMultimap.Builder<>(); @@ -319,7 +320,7 @@ public class TaskQueueHelper { inputParams.putAll(UriParameters.parse(query)); } if (headers.containsEntry( - HttpHeaders.CONTENT_TYPE.toLowerCase(), MediaType.FORM_DATA.toString())) { + Ascii.toLowerCase(HttpHeaders.CONTENT_TYPE), MediaType.FORM_DATA.toString())) { inputParams.putAll(UriParameters.parse(info.getBody())); } this.params = inputParams.build(); diff --git a/javatests/google/registry/tools/BUILD b/javatests/google/registry/tools/BUILD index 9583f12ff..f01c4d359 100644 --- a/javatests/google/registry/tools/BUILD +++ b/javatests/google/registry/tools/BUILD @@ -13,7 +13,9 @@ java_library( srcs = glob([ "*.java", ]), - resources = glob(["testdata/*.*"]), + resources = glob([ + "testdata/*.*", + ]), deps = [ "//java/com/google/common/annotations", "//java/com/google/common/base", diff --git a/javatests/google/registry/tools/CommandTestCase.java b/javatests/google/registry/tools/CommandTestCase.java index 80ec46f14..6dac241dc 100644 --- a/javatests/google/registry/tools/CommandTestCase.java +++ b/javatests/google/registry/tools/CommandTestCase.java @@ -49,7 +49,8 @@ import org.mockito.runners.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public abstract class CommandTestCase { - private ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + private final ByteArrayOutputStream stderr = new ByteArrayOutputStream(); protected C command; @@ -71,6 +72,7 @@ public abstract class CommandTestCase { RegistryToolEnvironment.UNITTEST.setup(); command = newCommandInstance(); System.setOut(new PrintStream(stdout)); + System.setErr(new PrintStream(stderr)); } void runCommandInEnvironment(RegistryToolEnvironment env, String... args) throws Exception { @@ -145,17 +147,29 @@ public abstract class CommandTestCase { return ofy().load().type(PollMessage.class).count(); } + protected void assertStdoutIs(String expected) throws Exception { + assertThat(getStdoutAsString()).isEqualTo(expected); + } + protected void assertInStdout(String... expected) throws Exception { + String stdout = getStdoutAsString(); for (String line : expected) { - assertThat(stdout.toString(UTF_8.toString())).contains(line); + assertThat(stdout).contains(line); + } + } + + protected void assertInStderr(String... expected) throws Exception { + String stderror = new String(stderr.toByteArray(), UTF_8); + for (String line : expected) { + assertThat(stderror).contains(line); } } void assertNotInStdout(String expected) throws Exception { - assertThat(stdout.toString(UTF_8.toString())).doesNotContain(expected); + assertThat(getStdoutAsString()).doesNotContain(expected); } - String getStdoutAsString() { + protected String getStdoutAsString() { return new String(stdout.toByteArray(), UTF_8); } diff --git a/python/google/registry/reporting/BUILD b/python/google/registry/reporting/BUILD new file mode 100644 index 000000000..4f4c7e073 --- /dev/null +++ b/python/google/registry/reporting/BUILD @@ -0,0 +1,18 @@ +package(default_visibility = ["//java/google/registry:registry_project"]) + +licenses(["notice"]) # Apache 2.0 + + +py_library( + name = "icann_report_query_builder", + srcs = ["icann_report_query_builder.py"], + deps = ["//python:python_directory_import"], +) + +py_test( + name = "icann_report_query_builder_test", + size = "small", + srcs = ["icann_report_query_builder_test.py"], + data = ["testdata/golden_activity_query.sql"], + deps = [":icann_report_query_builder"], +) diff --git a/python/google/registry/reporting/icann_report_query_builder.py b/python/google/registry/reporting/icann_report_query_builder.py new file mode 100644 index 000000000..b1f17bd74 --- /dev/null +++ b/python/google/registry/reporting/icann_report_query_builder.py @@ -0,0 +1,392 @@ +"""ICANN reporting BigQuery query construction logic. + +The IcannReportQueryBuilder class contains logic for constructing the +multi-part BigQuery queries used to produce ICANN monthly reports. These +queries are fairly complicated; see the design doc published to the +domain-registry-users@googlegroups.com for an overview. + +Currently, this class only supports building the query for activity +reports (not transaction reports). +""" +import datetime + +# This regex pattern matches the full signature of the 'EPP Command' log line +# from FlowRunner.run(), i.e. it matches the logging class/method that prefixes +# the log message, plus the 'EPP Command' string, up to the newline. +# Queries used below depend on matching this log line and parsing its +# exact format, so it must be kept in sync with the logging site. +# TODO(b/20725722): make the log statement format more robust. +FLOWRUNNER_LOG_SIGNATURE_PATTERN = '(?:{}): EPP Command'.format('|'.join([ + 'com.google.domain.registry.flows.FlowRunner run', + # TODO(b/29397966): figure out why this is FormattingLogger vs FlowRunner. + 'com.google.domain.registry.util.FormattingLogger log', + 'google.registry.util.FormattingLogger log'])) + + +class IcannReportQueryBuilder(object): + """Container for methods to build BigQuery queries for ICANN reporting.""" + + def BuildActivityReportQuery(self, month, registrar_count): + """Returns the assembled activity report query for a given month. + + Specifically, we instantiate the outermost activity report query by pointing + it at the union of a series of "data source" queries that each produce data + used to generate certain metrics. These queries in turn rely on some common + lower-level data source queries (monthly logs, both raw and EPP-parsed). + + Args: + month: (str) month of the report to generate, in YYYY-MM format + registrar_count: (int) total number of registrars in the registry system + + Returns: + (str) the fully-instantiated activity report query SQL + """ + # Construct some date-related parameters from the given month. + this_month_date = datetime.datetime.strptime(month, '%Y-%m').date() + # Hacky way to compute the start of the next month - add enough days to get + # to the next month (e.g. 31), then set the day to 1. It'd be cleaner to + # use dateutils.relativedelta(months=1) but the dependency is a pain. + month_delta = datetime.timedelta(days=31) + next_month_date = (this_month_date + month_delta).replace(day=1) + this_yearmonth = this_month_date.strftime('%Y-%m') + next_yearmonth = next_month_date.strftime('%Y-%m') + + # Construct the queries themselves. + logs_query = self._MakeMonthlyLogsQuery(this_yearmonth, next_yearmonth) + epp_xml_logs_query = self._MakeEppXmlLogsQuery(logs_query) + data_source_queries = [ + self._MakeActivityOperationalRegistrarsQuery(next_yearmonth), + self._MakeActivityAllRampedUpRegistrarsQuery(next_yearmonth), + self._MakeActivityAllRegistrarsQuery(registrar_count), + self._MakeActivityWhoisQuery(logs_query), self._MakeActivityDnsQuery(), + self._MakeActivityEppSrsMetricsQuery(epp_xml_logs_query) + ] + return _StripTrailingWhitespaceFromLines(self._MakeActivityReportQuery( + data_source_queries)) + + def _MakeMonthlyLogsQuery(self, this_yearmonth, next_yearmonth): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query AppEngine request logs for the report month. + SELECT + protoPayload.resource AS requestPath, + protoPayload.line.logMessage AS logMessage, + FROM + TABLE_DATE_RANGE_STRICT( + [appengine_logs.appengine_googleapis_com_request_log_], + TIMESTAMP('%(this_yearmonth)s-01'), + -- End timestamp is inclusive, so subtract 1 second from the + -- timestamp representing the start of the next month. + DATE_ADD(TIMESTAMP('%(next_yearmonth)s-01'), -1, 'SECOND')) + """ + return query % {'this_yearmonth': this_yearmonth, + 'next_yearmonth': next_yearmonth} + + def _MakeEppXmlLogsQuery(self, logs_query): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + # This query relies on regex-parsing the precise format of the 'EPP Command' + # log line from FlowRunner.run(), so it must be kept in sync. + # TODO(b/20725722): make the log statement format more robust. + query = r""" + -- Query EPP request logs and extract the clientId and raw EPP XML. + SELECT + REGEXP_EXTRACT(logMessage, r'^%(log_signature)s\n\t.+\n\t(.+)\n') AS clientId, + REGEXP_EXTRACT(logMessage, r'^%(log_signature)s\n\t.+\n\t.+\n\t.+\n\t((?s).+)$') AS xml, + FROM ( + -- BEGIN LOGS QUERY -- + %(logs_query)s + -- END LOGS QUERY -- + ) + WHERE + -- EPP endpoints from the proxy, regtool, and console respectively. + requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr') + AND REGEXP_MATCH(logMessage, r'^%(log_signature)s') + """ + return query % {'logs_query': logs_query, + 'log_signature': FLOWRUNNER_LOG_SIGNATURE_PATTERN} + + def _MakeActivityReportQuery(self, data_source_queries): + """Make the overall activity report query. + + Args: + data_source_queries: list of BigQuery SQL strings to use + as source 'tables' for the main query; each of these + queries must output a schema as follows: + + STRING tld / STRING metricName / INTEGER count + + A null TLD indicates that the metric counts towards + all TLDs. + + Returns: + query as a string of BigQuery SQL + """ + query = r""" + SELECT + Tld.tld AS tld, + SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars, + -- Compute ramp-up-registrars as all-ramped-up-registrars + -- minus operational-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'operational-registrars' THEN -count + WHEN metricName = 'all-ramped-up-registrars' THEN count + ELSE 0 + END)) AS ramp_up_registrars, + -- Compute pre-ramp-up-registrars as all-registrars minus + -- all-ramp-up-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'all-ramped-up-registrars' THEN -count + WHEN metricName = 'all-registrars' THEN count + ELSE 0 + END)) AS pre_ramp_up_registrars, + -- We don't support ZFA over SFTP, only AXFR. + 0 AS zfa_passwords, + SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries, + SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries, + -- We don't support searchable WHOIS. + 0 AS searchable_whois_queries, + -- DNS queries for UDP/TCP are all assumed to be recevied/responded. + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received, + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded, + -- SRS metrics. + SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check, + SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create, + SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete, + SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info, + SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew, + SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report, + SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request, + SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve, + SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel, + SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query, + SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject, + SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request, + SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update, + SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check, + SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create, + SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete, + SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info, + SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update, + SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check, + SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create, + SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete, + SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info, + SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve, + SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel, + SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query, + SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject, + SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request, + SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update, + -- Cross join a list of all TLDs against TLD-specific metrics and then + -- filter so that only metrics with that TLD or a NULL TLD are counted + -- towards a given TLD. + FROM ( + SELECT tldStr AS tld + FROM [latest_snapshot.Registry] + -- Include all real TLDs that are not in pre-delegation testing. + WHERE tldType = 'REAL' + OMIT RECORD IF SOME(tldStateTransitions.tldState = 'PDT') + ) AS Tld + CROSS JOIN ( + SELECT + tld, metricName, count + FROM + -- Dummy data source that ensures that all TLDs appear in report, + -- since they'll all have at least 1 joined row that survives. + (SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 AS count), + -- BEGIN JOINED DATA SOURCES -- + %(joined_data_sources)s + -- END JOINED DATA SOURCES -- + ) AS TldMetrics + WHERE Tld.tld = TldMetrics.tld OR TldMetrics.tld IS NULL + GROUP BY tld + ORDER BY tld + """ + # Turn each data source query into a subquery in parentheses, and join + # them together with comments (representing a table union). + joined_data_sources = '\n' + ',\n'.join( + '(\n%s\n)' % query for query in data_source_queries) + return query % {'joined_data_sources': joined_data_sources} + + def _MakeActivityOperationalRegistrarsQuery(self, next_yearmonth): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for operational-registrars metric. + SELECT + allowedTlds AS tld, + 'operational-registrars' AS metricName, + INTEGER(COUNT(__key__.name)) AS count, + FROM [domain-registry:latest_snapshot.Registrar] + WHERE type = 'REAL' + AND creationTime < TIMESTAMP('%(next_yearmonth)s-01') + GROUP BY tld + """ + return query % {'next_yearmonth': next_yearmonth} + + def _MakeActivityAllRampedUpRegistrarsQuery(self, next_yearmonth): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for all-ramped-up-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-ramped-up-registrars' AS metricName, + -- Sandbox OT&E registrar names can have either '-{1,2,3,4}' or '{,2,3}' + -- as suffixes - strip all of these off to get the "real" name. + INTEGER(EXACT_COUNT_DISTINCT( + REGEXP_EXTRACT(__key__.name, r'(.+?)(?:-?\d)?$'))) AS count, + FROM [domain-registry-sandbox:latest_snapshot.Registrar] + WHERE type = 'OTE' + AND creationTime < TIMESTAMP('%(next_yearmonth)s-01') + """ + return query % {'next_yearmonth': next_yearmonth} + + def _MakeActivityAllRegistrarsQuery(self, registrar_count): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = """ + -- Query for all-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-registrars' AS metricName, + INTEGER('%(registrar_count)s') AS count, + """ + return query % {'registrar_count': registrar_count} + + def _MakeActivityWhoisQuery(self, logs_query): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for WHOIS metrics. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + -- Whois queries over port 43 get forwarded by the proxy to /_dr/whois, + -- while web queries come in via /whois/. + CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries' + WHEN LEFT(requestPath, 7) = '/whois/' THEN 'web-whois-queries' + END AS metricName, + INTEGER(COUNT(requestPath)) AS count, + FROM ( + -- BEGIN LOGS QUERY -- + %(logs_query)s + -- END LOGS QUERY -- + ) + GROUP BY metricName + HAVING metricName IS NOT NULL + """ + return query % {'logs_query': logs_query} + + def _MakeActivityDnsQuery(self): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for DNS metrics. + SELECT + STRING(NULL) AS tld, + metricName, + -1 AS count, + FROM + (SELECT 'dns-udp-queries' AS metricName), + (SELECT 'dns-tcp-queries' AS metricName) + """ + return query + + def _MakeActivityEppSrsMetricsQuery(self, epp_xml_logs_query): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query EPP XML messages and calculate SRS metrics. + SELECT + domainTld AS tld, + -- SRS metric names follow a set pattern corresponding to the EPP + -- protocol elements. First we extract the 'inner' command element in + -- EPP, e.g. , which is the resource type followed by + -- the standard EPP command type. To get the metric name, we add the + -- prefix 'srs-', abbreviate 'domain' as 'dom' and 'contact' as 'cont', + -- and replace ':' with '-' to produce 'srs-dom-create'. + -- + -- Transfers have subcommands indicated by an 'op' attribute, which we + -- extract and add as an extra suffix for transfer commands, so e.g. + -- 'srs-cont-transfer-approve'. Domain restores are domain updates + -- with a special element; if present, the command counts + -- under the srs-dom-rgp-restore-{request,report} metric (depending on + -- the value of the 'op' attribute) instead of srs-dom-update. + CONCAT( + 'srs-', + REPLACE(REPLACE(REPLACE( + CASE + WHEN NOT restoreOp IS NULL THEN CONCAT('domain-rgp-restore-', restoreOp) + WHEN commandType = 'transfer' THEN CONCAT(innerCommand, '-', commandOpArg) + ELSE innerCommand + END, + ':', '-'), 'domain', 'dom'), 'contact', 'cont') + ) AS metricName, + INTEGER(COUNT(xml)) AS count, + FROM ( + SELECT + -- Extract salient bits of the EPP XML using regexes. This is fairly + -- safe since the EPP gets schema-validated and pretty-printed before + -- getting logged, and so it looks something like this: + -- + -- + -- + -- , 'request' as the value of the + -- 'op' attribute of that element (if any), and 'domain:transfer' as + -- the inner command from the name of the subsequent element. + -- + -- Domain commands all have at least one element (more + -- than one for domain checks, but we just count the first), from which + -- we extract the domain TLD as everything after the first dot in the + -- element value. This won't work if the client mistakenly sends a + -- hostname (e.g. 'www.foo.example') as the domain name, but we prefer + -- this over taking everything after the last dot so that multipart + -- TLDs like 'co.uk' can be supported. + -- + -- Domain restores are indicated by an element, from + -- which we extract the value of the 'op' attribute. + -- + -- TODO(b/20725722): preprocess the XML in FlowRunner so we don't need + -- regex parsing of XML here (http://stackoverflow.com/a/1732454). + -- + REGEXP_EXTRACT(xml, '(?s).*?<([a-z]+)') AS commandType, + REGEXP_EXTRACT(xml, '(?s).*?<[a-z]+ op="(.+?)"') AS commandOpArg, + REGEXP_EXTRACT(xml, '(?s).*?<.+?>.*?<([a-z]+:[a-z]+)') AS innerCommand, + REGEXP_EXTRACT(xml, '[^.]+[.](.+)') AS domainTld, + REGEXP_EXTRACT(xml, '') AS restoreOp, + xml, + FROM ( + -- BEGIN EPP XML LOGS QUERY -- + %(epp_xml_logs_query)s + -- END EPP XML LOGS QUERY -- + ) + -- Filter to just XML that contains a element (no s). + WHERE xml CONTAINS '' + ) + -- Whitelist of EPP command types that we care about for metrics; + -- excludes login, logout, and poll. + WHERE commandType IN ('check', 'create', 'delete', 'info', 'renew', 'transfer', 'update') + GROUP BY tld, metricName + """ + return query % {'epp_xml_logs_query': epp_xml_logs_query} + + +def _StripTrailingWhitespaceFromLines(string): + """Strips trailing whitespace from each line of the provided string. + + Args: + string: (str) string to remove trailing whitespace from + + Returns: + (str) input string, with trailing whitespace stripped from each line + """ + return '\n'.join(line.rstrip() for line in string.split('\n')) diff --git a/python/google/registry/reporting/icann_report_query_builder_test.py b/python/google/registry/reporting/icann_report_query_builder_test.py new file mode 100644 index 000000000..32c595d50 --- /dev/null +++ b/python/google/registry/reporting/icann_report_query_builder_test.py @@ -0,0 +1,44 @@ +"""Tests for google.registry.reporting.icann_report_query_builder.""" + +import os +import unittest + +from google.registry.reporting import icann_report_query_builder + + +class IcannReportQueryBuilderTest(unittest.TestCase): + + testdata_path = None + + def setUp(self): + # Using __file__ is a bit of a hack, but it's the only way that "just works" + # for internal and external versions of the code, and it's fine for tests. + self.testdata_path = os.path.join(os.path.dirname(__file__), 'testdata') + + def testActivityQuery_matchesGoldenQuery(self): + self.maxDiff = None # Show long diffs + query_builder = icann_report_query_builder.IcannReportQueryBuilder() + golden_activity_query_path = os.path.join(self.testdata_path, + 'golden_activity_query.sql') + with open(golden_activity_query_path, 'r') as golden_activity_query: + self.assertMultiLineEqual(golden_activity_query.read(), + query_builder.BuildActivityReportQuery( + month='2016-06', + registrar_count=None)) + + def testStringTrailingWhitespaceFromLines(self): + def do_test(expected, original): + self.assertEqual( + expected, + icann_report_query_builder._StripTrailingWhitespaceFromLines( + original)) + do_test('foo\nbar\nbaz\n', 'foo\nbar\nbaz\n') + do_test('foo\nbar\nbaz\n', 'foo \nbar \nbaz \n') + do_test('foo\nbar\nbaz', 'foo \nbar \nbaz ') + do_test('\nfoo\nbar\nbaz', '\nfoo\nbar\nbaz') + do_test('foo\n\n', 'foo\n \n') + do_test('foo\n', 'foo\n ') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/registry/reporting/testdata/golden_activity_query.sql b/python/google/registry/reporting/testdata/golden_activity_query.sql new file mode 100644 index 000000000..4e7318dc8 --- /dev/null +++ b/python/google/registry/reporting/testdata/golden_activity_query.sql @@ -0,0 +1,269 @@ + + SELECT + Tld.tld AS tld, + SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars, + -- Compute ramp-up-registrars as all-ramped-up-registrars + -- minus operational-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'operational-registrars' THEN -count + WHEN metricName = 'all-ramped-up-registrars' THEN count + ELSE 0 + END)) AS ramp_up_registrars, + -- Compute pre-ramp-up-registrars as all-registrars minus + -- all-ramp-up-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'all-ramped-up-registrars' THEN -count + WHEN metricName = 'all-registrars' THEN count + ELSE 0 + END)) AS pre_ramp_up_registrars, + -- We don't support ZFA over SFTP, only AXFR. + 0 AS zfa_passwords, + SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries, + SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries, + -- We don't support searchable WHOIS. + 0 AS searchable_whois_queries, + -- DNS queries for UDP/TCP are all assumed to be recevied/responded. + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received, + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded, + -- SRS metrics. + SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check, + SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create, + SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete, + SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info, + SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew, + SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report, + SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request, + SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve, + SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel, + SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query, + SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject, + SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request, + SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update, + SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check, + SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create, + SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete, + SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info, + SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update, + SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check, + SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create, + SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete, + SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info, + SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve, + SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel, + SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query, + SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject, + SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request, + SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update, + -- Cross join a list of all TLDs against TLD-specific metrics and then + -- filter so that only metrics with that TLD or a NULL TLD are counted + -- towards a given TLD. + FROM ( + SELECT tldStr AS tld + FROM [latest_snapshot.Registry] + -- Include all real TLDs that are not in pre-delegation testing. + WHERE tldType = 'REAL' + OMIT RECORD IF SOME(tldStateTransitions.tldState = 'PDT') + ) AS Tld + CROSS JOIN ( + SELECT + tld, metricName, count + FROM + -- Dummy data source that ensures that all TLDs appear in report, + -- since they'll all have at least 1 joined row that survives. + (SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 AS count), + -- BEGIN JOINED DATA SOURCES -- + +( + + -- Query for operational-registrars metric. + SELECT + allowedTlds AS tld, + 'operational-registrars' AS metricName, + INTEGER(COUNT(__key__.name)) AS count, + FROM [domain-registry:latest_snapshot.Registrar] + WHERE type = 'REAL' + AND creationTime < TIMESTAMP('2016-07-01') + GROUP BY tld + +), +( + + -- Query for all-ramped-up-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-ramped-up-registrars' AS metricName, + -- Sandbox OT&E registrar names can have either '-{1,2,3,4}' or '{,2,3}' + -- as suffixes - strip all of these off to get the "real" name. + INTEGER(EXACT_COUNT_DISTINCT( + REGEXP_EXTRACT(__key__.name, r'(.+?)(?:-?\d)?$'))) AS count, + FROM [domain-registry-sandbox:latest_snapshot.Registrar] + WHERE type = 'OTE' + AND creationTime < TIMESTAMP('2016-07-01') + +), +( + + -- Query for all-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-registrars' AS metricName, + INTEGER('None') AS count, + +), +( + + -- Query for WHOIS metrics. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + -- Whois queries over port 43 get forwarded by the proxy to /_dr/whois, + -- while web queries come in via /whois/. + CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries' + WHEN LEFT(requestPath, 7) = '/whois/' THEN 'web-whois-queries' + END AS metricName, + INTEGER(COUNT(requestPath)) AS count, + FROM ( + -- BEGIN LOGS QUERY -- + + -- Query AppEngine request logs for the report month. + SELECT + protoPayload.resource AS requestPath, + protoPayload.line.logMessage AS logMessage, + FROM + TABLE_DATE_RANGE_STRICT( + [appengine_logs.appengine_googleapis_com_request_log_], + TIMESTAMP('2016-06-01'), + -- End timestamp is inclusive, so subtract 1 second from the + -- timestamp representing the start of the next month. + DATE_ADD(TIMESTAMP('2016-07-01'), -1, 'SECOND')) + + -- END LOGS QUERY -- + ) + GROUP BY metricName + HAVING metricName IS NOT NULL + +), +( + + -- Query for DNS metrics. + SELECT + STRING(NULL) AS tld, + metricName, + -1 AS count, + FROM + (SELECT 'dns-udp-queries' AS metricName), + (SELECT 'dns-tcp-queries' AS metricName) + +), +( + + -- Query EPP XML messages and calculate SRS metrics. + SELECT + domainTld AS tld, + -- SRS metric names follow a set pattern corresponding to the EPP + -- protocol elements. First we extract the 'inner' command element in + -- EPP, e.g. , which is the resource type followed by + -- the standard EPP command type. To get the metric name, we add the + -- prefix 'srs-', abbreviate 'domain' as 'dom' and 'contact' as 'cont', + -- and replace ':' with '-' to produce 'srs-dom-create'. + -- + -- Transfers have subcommands indicated by an 'op' attribute, which we + -- extract and add as an extra suffix for transfer commands, so e.g. + -- 'srs-cont-transfer-approve'. Domain restores are domain updates + -- with a special element; if present, the command counts + -- under the srs-dom-rgp-restore-{request,report} metric (depending on + -- the value of the 'op' attribute) instead of srs-dom-update. + CONCAT( + 'srs-', + REPLACE(REPLACE(REPLACE( + CASE + WHEN NOT restoreOp IS NULL THEN CONCAT('domain-rgp-restore-', restoreOp) + WHEN commandType = 'transfer' THEN CONCAT(innerCommand, '-', commandOpArg) + ELSE innerCommand + END, + ':', '-'), 'domain', 'dom'), 'contact', 'cont') + ) AS metricName, + INTEGER(COUNT(xml)) AS count, + FROM ( + SELECT + -- Extract salient bits of the EPP XML using regexes. This is fairly + -- safe since the EPP gets schema-validated and pretty-printed before + -- getting logged, and so it looks something like this: + -- + -- + -- + -- , 'request' as the value of the + -- 'op' attribute of that element (if any), and 'domain:transfer' as + -- the inner command from the name of the subsequent element. + -- + -- Domain commands all have at least one element (more + -- than one for domain checks, but we just count the first), from which + -- we extract the domain TLD as everything after the first dot in the + -- element value. This won't work if the client mistakenly sends a + -- hostname (e.g. 'www.foo.example') as the domain name, but we prefer + -- this over taking everything after the last dot so that multipart + -- TLDs like 'co.uk' can be supported. + -- + -- Domain restores are indicated by an element, from + -- which we extract the value of the 'op' attribute. + -- + -- TODO(b/20725722): preprocess the XML in FlowRunner so we don't need + -- regex parsing of XML here (http://stackoverflow.com/a/1732454). + -- + REGEXP_EXTRACT(xml, '(?s).*?<([a-z]+)') AS commandType, + REGEXP_EXTRACT(xml, '(?s).*?<[a-z]+ op="(.+?)"') AS commandOpArg, + REGEXP_EXTRACT(xml, '(?s).*?<.+?>.*?<([a-z]+:[a-z]+)') AS innerCommand, + REGEXP_EXTRACT(xml, '[^.]+[.](.+)') AS domainTld, + REGEXP_EXTRACT(xml, '') AS restoreOp, + xml, + FROM ( + -- BEGIN EPP XML LOGS QUERY -- + + -- Query EPP request logs and extract the clientId and raw EPP XML. + SELECT + REGEXP_EXTRACT(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command\n\t.+\n\t(.+)\n') AS clientId, + REGEXP_EXTRACT(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command\n\t.+\n\t.+\n\t.+\n\t((?s).+)$') AS xml, + FROM ( + -- BEGIN LOGS QUERY -- + + -- Query AppEngine request logs for the report month. + SELECT + protoPayload.resource AS requestPath, + protoPayload.line.logMessage AS logMessage, + FROM + TABLE_DATE_RANGE_STRICT( + [appengine_logs.appengine_googleapis_com_request_log_], + TIMESTAMP('2016-06-01'), + -- End timestamp is inclusive, so subtract 1 second from the + -- timestamp representing the start of the next month. + DATE_ADD(TIMESTAMP('2016-07-01'), -1, 'SECOND')) + + -- END LOGS QUERY -- + ) + WHERE + -- EPP endpoints from the proxy, regtool, and console respectively. + requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr') + AND REGEXP_MATCH(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command') + + -- END EPP XML LOGS QUERY -- + ) + -- Filter to just XML that contains a element (no s). + WHERE xml CONTAINS '' + ) + -- Whitelist of EPP command types that we care about for metrics; + -- excludes login, logout, and poll. + WHERE commandType IN ('check', 'create', 'delete', 'info', 'renew', 'transfer', 'update') + GROUP BY tld, metricName + +) + -- END JOINED DATA SOURCES -- + ) AS TldMetrics + WHERE Tld.tld = TldMetrics.tld OR TldMetrics.tld IS NULL + GROUP BY tld + ORDER BY tld