zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-07-24 19:48:32 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge in latest changes (fix some Markdown doc conflicts)

Browse source
This commit is contained in:
Ben McIlwain 2016-07-19 10:50:01 -04:00
commit f75bb65fd3
36 changed files with 1124 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

3
java/google/registry/config/RegistryEnvironment.java
View file

@ -15,6 +15,7 @@
package google.registry.config;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
import javax.annotation.Nullable;
/** Registry environments. */
@ -50,7 +51,7 @@ public enum RegistryEnvironment {
/** Returns environment configured by system property {@value #PROPERTY}. */
public static RegistryEnvironment get() {
return valueOf(System.getProperty(PROPERTY, UNITTEST.name()).toUpperCase());
return valueOf(Ascii.toUpperCase(System.getProperty(PROPERTY, UNITTEST.name())));
}
/**

3
java/google/registry/export/DatastoreBackupInfo.java
View file

@ -21,6 +21,7 @@ import static org.joda.time.DateTimeZone.UTC;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.Text;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ascii;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
@ -140,7 +141,7 @@ public class DatastoreBackupInfo {
"Status: " + getStatus(),
"Started: " + startTime,
"Ended: " + completeTime.orNull(),
"Duration: " + getRunningTime().toPeriod().toString().substring(2).toLowerCase(),
"Duration: " + Ascii.toLowerCase(getRunningTime().toPeriod().toString().substring(2)),
"GCS: " + gcsFilename.orNull(),
"Kinds: " + kinds,
"");

3
java/google/registry/model/domain/fee/FeeCommandDescriptor.java
View file

@ -14,6 +14,7 @@
package google.registry.model.domain.fee;
import com.google.common.base.Ascii;
import com.google.common.base.CharMatcher;
import google.registry.model.ImmutableObject;
import javax.xml.bind.annotation.XmlAttribute;
@ -56,7 +57,7 @@ public class FeeCommandDescriptor extends ImmutableObject {
// Require the xml string to be lowercase.
if (command != null && CharMatcher.javaLowerCase().matchesAllOf(command)) {
try {
return CommandName.valueOf(command.toUpperCase());
return CommandName.valueOf(Ascii.toUpperCase(command));
} catch (IllegalArgumentException e) {
// Swallow this and return UNKNOWN below because there's no matching CommandName.
}

3
java/google/registry/model/domain/launch/LaunchNotice.java
View file

@ -20,6 +20,7 @@ import static com.google.common.io.BaseEncoding.base16;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import com.google.common.base.Ascii;
import com.google.common.base.CharMatcher;
import com.google.common.base.Optional;
import com.google.common.primitives.Ints;
@ -101,7 +102,7 @@ public class LaunchNotice extends ImmutableObject {
String tcnId = getNoticeId().getTcnId();
checkArgument(tcnId.length() == 27);
int checksum = Ints.fromByteArray(base16().decode(tcnId.substring(0, 8).toUpperCase()));
int checksum = Ints.fromByteArray(base16().decode(Ascii.toUpperCase(tcnId.substring(0, 8))));
String noticeId = tcnId.substring(8);
checkArgument(CharMatcher.inRange('0', '9').matchesAllOf(noticeId));

53
java/google/registry/rdap/RdapJsonFormatter.java
View file

@ -85,8 +85,9 @@ public class RdapJsonFormatter {
static final String NOTICES = "notices";
private static final String REMARKS = "remarks";
/** Status values specified in RFC 7483 § 10.2.2. */
private enum RdapStatus {
// Status values specified in RFC 7483 § 10.2.2.
VALIDATED("validated"),
RENEW_PROHIBITED("renew prohibited"),
UPDATE_PROHIBITED("update prohibited"),
@ -104,7 +105,26 @@ public class RdapJsonFormatter {
PENDING_RENEW("pending renew"),
PENDING_TRANSFER("pending transfer"),
PENDING_UPDATE("pending update"),
PENDING_DELETE("pending delete");
PENDING_DELETE("pending delete"),
// Additional status values defined in
// https://tools.ietf.org/html/draft-ietf-regext-epp-rdap-status-mapping-01.
ADD_PERIOD("add period"),
AUTO_RENEW_PERIOD("auto renew period"),
CLIENT_DELETE_PROHIBITED("client delete prohibited"),
CLIENT_HOLD("client hold"),
CLIENT_RENEW_PROHIBITED("client renew prohibited"),
CLIENT_TRANSFER_PROHIBITED("client transfer prohibited"),
CLIENT_UPDATE_PROHIBITED("client update prohibited"),
PENDING_RESTORE("pending restore"),
REDEMPTION_PERIOD("redemption period"),
RENEW_PERIOD("renew period"),
SERVER_DELETE_PROHIBITED("server deleted prohibited"),
SERVER_RENEW_PROHIBITED("server renew prohibited"),
SERVER_TRANSFER_PROHIBITED("server transfer prohibited"),
SERVER_UPDATE_PROHIBITED("server update prohibited"),
SERVER_HOLD("server hold"),
TRANSFER_PERIOD("transfer period");
/** Value as it appears in RDAP messages. */
private final String rfc7483String;
@ -123,23 +143,30 @@ public class RdapJsonFormatter {
private static final ImmutableMap<StatusValue, RdapStatus> statusToRdapStatusMap =
Maps.immutableEnumMap(
new ImmutableMap.Builder<StatusValue, RdapStatus>()
.put(StatusValue.CLIENT_DELETE_PROHIBITED, RdapStatus.DELETE_PROHIBITED)
.put(StatusValue.CLIENT_HOLD, RdapStatus.INACTIVE)
.put(StatusValue.CLIENT_RENEW_PROHIBITED, RdapStatus.RENEW_PROHIBITED)
.put(StatusValue.CLIENT_TRANSFER_PROHIBITED, RdapStatus.TRANSFER_PROHIBITED)
.put(StatusValue.CLIENT_UPDATE_PROHIBITED, RdapStatus.UPDATE_PROHIBITED)
// StatusValue.ADD_PERIOD not defined in our system
// StatusValue.AUTO_RENEW_PERIOD not defined in our system
.put(StatusValue.CLIENT_DELETE_PROHIBITED, RdapStatus.CLIENT_DELETE_PROHIBITED)
.put(StatusValue.CLIENT_HOLD, RdapStatus.CLIENT_HOLD)
.put(StatusValue.CLIENT_RENEW_PROHIBITED, RdapStatus.CLIENT_RENEW_PROHIBITED)
.put(StatusValue.CLIENT_TRANSFER_PROHIBITED, RdapStatus.CLIENT_TRANSFER_PROHIBITED)
.put(StatusValue.CLIENT_UPDATE_PROHIBITED, RdapStatus.CLIENT_UPDATE_PROHIBITED)
.put(StatusValue.INACTIVE, RdapStatus.INACTIVE)
.put(StatusValue.LINKED, RdapStatus.ASSOCIATED)
.put(StatusValue.OK, RdapStatus.ACTIVE)
.put(StatusValue.PENDING_CREATE, RdapStatus.PENDING_CREATE)
.put(StatusValue.PENDING_DELETE, RdapStatus.PENDING_DELETE)
// StatusValue.PENDING_RENEW not defined in our system
// StatusValue.PENDING_RESTORE not defined in our system
.put(StatusValue.PENDING_TRANSFER, RdapStatus.PENDING_TRANSFER)
.put(StatusValue.PENDING_UPDATE, RdapStatus.PENDING_UPDATE)
.put(StatusValue.SERVER_DELETE_PROHIBITED, RdapStatus.DELETE_PROHIBITED)
.put(StatusValue.SERVER_HOLD, RdapStatus.INACTIVE)
.put(StatusValue.SERVER_RENEW_PROHIBITED, RdapStatus.RENEW_PROHIBITED)
.put(StatusValue.SERVER_TRANSFER_PROHIBITED, RdapStatus.TRANSFER_PROHIBITED)
.put(StatusValue.SERVER_UPDATE_PROHIBITED, RdapStatus.UPDATE_PROHIBITED)
// StatusValue.REDEMPTION_PERIOD not defined in our system
// StatusValue.RENEW_PERIOD not defined in our system
.put(StatusValue.SERVER_DELETE_PROHIBITED, RdapStatus.SERVER_DELETE_PROHIBITED)
.put(StatusValue.SERVER_HOLD, RdapStatus.SERVER_HOLD)
.put(StatusValue.SERVER_RENEW_PROHIBITED, RdapStatus.SERVER_RENEW_PROHIBITED)
.put(StatusValue.SERVER_TRANSFER_PROHIBITED, RdapStatus.SERVER_TRANSFER_PROHIBITED)
.put(StatusValue.SERVER_UPDATE_PROHIBITED, RdapStatus.SERVER_UPDATE_PROHIBITED)
// StatusValue.TRANSFER_PERIOD not defined in our system
.build());
/** Role values specified in RFC 7483 § 10.2.4. */
@ -189,7 +216,7 @@ public class RdapJsonFormatter {
}
}
/** Map of EPP status values to the RDAP equivalents. */
/** Map of EPP event values to the RDAP equivalents. */
private static final ImmutableMap<HistoryEntry.Type, RdapEventAction>
historyEntryTypeToRdapEventActionMap =
Maps.immutableEnumMap(

3
java/google/registry/rde/DomainResourceToXjcConverter.java
View file

@ -14,6 +14,7 @@
package google.registry.rde;
import com.google.common.base.Ascii;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableSet;
import com.googlecode.objectify.Ref;
@ -280,7 +281,7 @@ final class DomainResourceToXjcConverter {
private static XjcDomainContactType convertDesignatedContact(DesignatedContact model) {
XjcDomainContactType bean = new XjcDomainContactType();
ContactResource contact = model.getContactRef().get();
bean.setType(XjcDomainContactAttrType.fromValue(model.getType().toString().toLowerCase()));
bean.setType(XjcDomainContactAttrType.fromValue(Ascii.toLowerCase(model.getType().toString())));
bean.setValue(contact.getContactId());
return bean;
}

101
java/google/registry/rde/RdeImportUtils.java Normal file
View file

@ -0,0 +1,101 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde;
import static com.google.common.base.Preconditions.checkState;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Work;
import google.registry.model.contact.ContactResource;
import google.registry.model.index.EppResourceIndex;
import google.registry.model.index.ForeignKeyIndex;
import google.registry.model.ofy.Ofy;
import google.registry.util.Clock;
import google.registry.util.FormattingLogger;
import javax.inject.Inject;
/** Utility functions for escrow file import. */
public final class RdeImportUtils {
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
private final Ofy ofy;
private final Clock clock;
@Inject
public RdeImportUtils(Ofy ofy, Clock clock) {
this.ofy = ofy;
this.clock = clock;
}
/**
* Imports a contact from an escrow file.
*
* <p>The contact will only be imported if it has not been previously imported.
*
* <p>If the contact is imported, {@link ForeignKeyIndex} and {@link EppResourceIndex} are also
* created.
*
* @return true if the contact was created or updated, false otherwise.
*/
public boolean importContact(final ContactResource resource) {
return ofy.transact(
new Work<Boolean>() {
@Override
public Boolean run() {
ContactResource existing = ofy.load().key(Key.create(resource)).now();
if (existing == null) {
ForeignKeyIndex<ContactResource> existingForeignKeyIndex =
ForeignKeyIndex.load(
ContactResource.class, resource.getContactId(), clock.nowUtc());
// foreign key index should not exist, since existing contact was not found.
checkState(
existingForeignKeyIndex == null,
String.format(
"New contact resource has existing foreign key index. "
+ "contactId=%s, repoId=%s",
resource.getContactId(), resource.getRepoId()));
ofy.save().entity(resource);
ofy.save().entity(ForeignKeyIndex.create(resource, resource.getDeletionTime()));
ofy.save().entity(EppResourceIndex.create(Key.create(resource)));
logger.infofmt(
"Imported contact resource - ROID=%s, id=%s",
resource.getRepoId(), resource.getContactId());
return true;
} else if (!existing.getRepoId().equals(resource.getRepoId())) {
logger.warningfmt(
"Existing contact with same contact id but different ROID. "
+ "contactId=%s, existing ROID=%s, new ROID=%s",
resource.getContactId(), existing.getRepoId(), resource.getRepoId());
}
return false;
}
});
}
}

13
java/google/registry/tmch/LordnLog.java
View file

@ -18,6 +18,7 @@ import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import com.google.common.base.Ascii;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import com.google.re2j.Pattern;
@ -218,7 +219,7 @@ public final class LordnLog implements Iterable<Entry<String, LordnLog.Result>>
// + <Status flag>, whether the LORDN file has been accepted for
// processing by the TMDB. Possible values are "accepted" or
// "rejected".
Status status = Status.valueOf(firstLine.get(4).toUpperCase());
Status status = Status.valueOf(Ascii.toUpperCase(firstLine.get(4)));
// + <Warning flag>, whether the LORDN Log has any warning result
// codes. Possible values are "no-warnings" or "warnings-
@ -229,8 +230,11 @@ public final class LordnLog implements Iterable<Entry<String, LordnLog.Result>>
// processed in the LORDN file.
int dnLines = Integer.parseInt(firstLine.get(6));
int actual = lines.size() - 2;
checkArgument(dnLines == actual,
"Line 1: Number of entries (%d) differs from declaration (%d)", actual, dnLines);
checkArgument(
dnLines == actual,
"Line 1: Number of entries (%s) differs from declaration (%s)",
String.valueOf(actual),
String.valueOf(dnLines));
// Second line contains headers: roid,result-code
checkArgument(lines.get(1).equals("roid,result-code"),
@ -244,8 +248,7 @@ public final class LordnLog implements Iterable<Entry<String, LordnLog.Result>>
"Line %d: Expected 2 elements, found %d", i + 1, currentLine.size()));
String roid = currentLine.get(0);
int code = Integer.parseInt(currentLine.get(1));
Result result = checkNotNull(RESULTS.get(code),
"Line %d: Unknown result code: %d", i, code);
Result result = checkNotNull(RESULTS.get(code), "Line %s: Unknown result code: %s", i, code);
builder.put(roid, result);
}

3
java/google/registry/tools/AllocateDomainCommand.java
View file

@ -26,6 +26,7 @@ import static google.registry.tools.CommandUtilities.addHeader;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Ascii;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
@ -134,7 +135,7 @@ final class AllocateDomainCommand extends MutatingEppToolCommand {
ImmutableMap.Builder<String, String> contactsMapBuilder = new ImmutableMap.Builder<>();
for (DesignatedContact contact : application.getContacts()) {
contactsMapBuilder.put(
contact.getType().toString().toLowerCase(),
Ascii.toLowerCase(contact.getType().toString()),
contact.getContactRef().get().getForeignKey());
}
LaunchNotice launchNotice = application.getLaunchNotice();

3
java/google/registry/tools/CommandUtilities.java
View file

@ -14,6 +14,7 @@
package google.registry.tools;
import com.google.common.base.Ascii;
import com.google.common.base.Strings;
/** Container class for static utility methods. */
@ -25,6 +26,6 @@ class CommandUtilities {
/** Prompts for yes/no input using promptText, defaulting to no. */
static boolean promptForYes(String promptText) {
return System.console().readLine(promptText + " (y/N): ").toUpperCase().startsWith("Y");
return Ascii.toUpperCase(System.console().readLine(promptText + " (y/N): ")).startsWith("Y");
}
}

3
java/google/registry/tools/ConvertIdnCommand.java
View file

@ -19,6 +19,7 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Ascii;
import google.registry.tools.Command.GtechCommand;
import google.registry.util.Idn;
import java.io.IOException;
@ -37,7 +38,7 @@ final class ConvertIdnCommand implements Command, GtechCommand {
public void run() throws IOException {
for (String label : mainParameters) {
if (label.startsWith(ACE_PREFIX)) {
System.out.println(Idn.toUnicode(label.toLowerCase()));
System.out.println(Idn.toUnicode(Ascii.toLowerCase(label)));
} else {
System.out.println(canonicalizeDomainName(label));
}

5
java/google/registry/tools/DomainApplicationInfoCommand.java
View file

@ -18,6 +18,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.google.common.base.Ascii;
import com.google.template.soy.data.SoyMapData;
import google.registry.model.domain.launch.LaunchPhase;
import google.registry.tools.Command.GtechCommand;
@ -53,8 +54,8 @@ final class DomainApplicationInfoCommand extends EppToolCommand implements Gtech
@Override
void initEppToolCommand() {
LaunchPhase launchPhase =
checkArgumentNotNull(LaunchPhase.fromValue(phase.toLowerCase()), "Illegal launch phase.");
LaunchPhase launchPhase = checkArgumentNotNull(
LaunchPhase.fromValue(Ascii.toLowerCase(phase)), "Illegal launch phase.");
setSoyTemplate(
DomainApplicationInfoSoyInfo.getInstance(),

3
java/google/registry/tools/RegistryToolEnvironment.java
View file

@ -17,6 +17,7 @@ package google.registry.tools;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import google.registry.config.RegistryEnvironment;
@ -61,7 +62,7 @@ enum RegistryToolEnvironment {
* @see #get()
*/
static RegistryToolEnvironment parseFromArgs(String[] args) {
return valueOf(getFlagValue(args, FLAGS).toUpperCase());
return valueOf(Ascii.toUpperCase(getFlagValue(args, FLAGS)));
}
/**

3
java/google/registry/tools/server/VerifyOteAction.java
View file

@ -23,6 +23,7 @@ import static google.registry.util.CollectionUtils.isNullOrEmpty;
import static google.registry.util.DomainNameUtils.ACE_PREFIX;
import static java.util.Arrays.asList;
import com.google.common.base.Ascii;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
@ -250,7 +251,7 @@ public class VerifyOteAction implements Runnable, JsonAction {
/** Returns a more human-readable translation of the enum constant. */
String description() {
return this.name().replace('_', ' ').toLowerCase();
return Ascii.toLowerCase(this.name().replace('_', ' '));
}
/** An {@link EppInput} might match multiple actions, so check if this action matches. */

12
java/google/registry/ui/forms/FormField.java
View file

@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.base.Ascii;
import com.google.common.base.CharMatcher;
import com.google.common.base.Function;
import com.google.common.base.Functions;
@ -30,6 +31,7 @@ import com.google.common.collect.Range;
import com.google.re2j.Pattern;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import javax.annotation.Detainted;
@ -560,7 +562,7 @@ public final class FormField<I, O> {
@Nullable
@Override
public String apply(@Nullable String input) {
return input != null ? input.toUpperCase() : null;
return input != null ? input.toUpperCase(Locale.ENGLISH) : null;
}};
private static final Function<String, String> LOWERCASE_FUNCTION =
@ -568,7 +570,7 @@ public final class FormField<I, O> {
@Nullable
@Override
public String apply(@Nullable String input) {
return input != null ? input.toLowerCase() : null;
return input != null ? input.toLowerCase(Locale.ENGLISH) : null;
}};
private static final Function<Object, Object> REQUIRED_FUNCTION =
@ -587,8 +589,8 @@ public final class FormField<I, O> {
@Nullable
@Override
public Object apply(@Nullable Object input) {
return input instanceof CharSequence && ((CharSequence) input).length() == 0
|| input instanceof Collection && ((Collection<?>) input).isEmpty()
return ((input instanceof CharSequence) && (((CharSequence) input).length() == 0))
|| ((input instanceof Collection) && ((Collection<?>) input).isEmpty())
? null : input;
}};
@ -709,7 +711,7 @@ public final class FormField<I, O> {
@Override
public C apply(@Nullable O input) {
try {
return input != null ? Enum.valueOf(enumClass, ((String) input).toUpperCase()) : null;
return input != null ? Enum.valueOf(enumClass, Ascii.toUpperCase((String) input)) : null;
} catch (IllegalArgumentException e) {
throw new FormFieldException(
String.format("Enum %s does not contain '%s'", enumClass.getSimpleName(), input));

3
java/google/registry/ui/server/RegistrarFormFields.java
View file

@ -19,6 +19,7 @@ import static com.google.common.collect.Range.atMost;
import static com.google.common.collect.Range.closed;
import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import com.google.common.base.Ascii;
import com.google.common.base.Function;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
@ -354,7 +355,7 @@ public final class RegistrarFormFields {
}
for (String state : stateField.extractUntyped(args).asSet()) {
if ("US".equals(countryCode)) {
state = state.toUpperCase();
state = Ascii.toUpperCase(state);
if (!StateCode.US_MAP.containsKey(state)) {
throw new FormFieldException(stateField, "Unknown US state code.");
}

4
java/google/registry/util/Idn.java
View file

@ -57,8 +57,8 @@ public final class Idn {
}
/**
* Translates a string from Unicode to ASCII Compatible Encoding (ACE), as defined by the ToASCII
* operation of <a href="http://www.ietf.org/rfc/rfc3490.txt">RFC 3490</a>.
* Translates a string from ASCII Compatible Encoding (ACE) to Unicode, as defined by the
* ToUnicode operation of <a href="http://www.ietf.org/rfc/rfc3490.txt">RFC 3490</a>.
*
* <p>This method always uses <a href="http://unicode.org/reports/tr46/">UTS46 transitional
* processing</a>.

6
java/google/registry/util/RegistrarUtils.java
View file

@ -16,11 +16,13 @@ package google.registry.util;
import static com.google.common.base.CharMatcher.javaLetterOrDigit;
import com.google.common.base.Ascii;
/** Utilities for working with {@code Registrar} objects. */
public class RegistrarUtils {
/** Strip out anything that isn't a letter or digit, and lowercase. */
public static String normalizeRegistrarName(String name) {
return javaLetterOrDigit().retainFrom(name).toLowerCase();
return Ascii.toLowerCase(javaLetterOrDigit().retainFrom(name));
}
/**
@ -29,6 +31,6 @@ public class RegistrarUtils {
* in Datastore, and is suitable for use in email addresses.
*/
public static String normalizeClientId(String clientId) {
return clientId.toLowerCase().replaceAll("[^a-z0-9\\-]", "");
return Ascii.toLowerCase(clientId).replaceAll("[^a-z0-9\\-]", "");
}
}

5
java/google/registry/util/UrlFetchUtils.java
View file

@ -26,6 +26,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.common.base.Ascii;
import com.google.common.base.Optional;
import com.google.common.net.MediaType;
import java.security.NoSuchAlgorithmException;
@ -49,9 +50,9 @@ public final class UrlFetchUtils {
}
private static Optional<String> getHeaderFirstInternal(Iterable<HTTPHeader> hdrs, String name) {
name = name.toLowerCase();
name = Ascii.toLowerCase(name);
for (HTTPHeader header : hdrs) {
if (header.getName().toLowerCase().equals(name)) {
if (Ascii.toLowerCase(header.getName()).equals(name)) {
return Optional.of(header.getValue());
}
}

4
javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java
View file

@ -259,12 +259,12 @@ public class DnsUpdateWriterTest {
Update update, String resourceName, int recordType, String... resourceData) {
ArrayList<String> expectedData = new ArrayList<>();
for (String resourceDatum : resourceData) {
expectedData.add(resourceDatum.toLowerCase());
expectedData.add(resourceDatum);
}
ArrayList<String> actualData = new ArrayList<>();
for (Record record : findUpdateRecords(update, resourceName, recordType)) {
actualData.add(record.rdataToString().toLowerCase());
actualData.add(record.rdataToString());
}
assertThat(actualData).containsExactlyElementsIn(expectedData);
}

5
javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java
View file

@ -26,6 +26,7 @@ import static google.registry.testing.DatastoreHelper.persistResource;
import static google.registry.testing.GenericEppResourceSubject.assertAboutEppResources;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableSet;
import com.googlecode.objectify.Ref;
import google.registry.flows.Flow;
@ -113,7 +114,7 @@ public class DomainTransferFlowTestCase<F extends Flow, R extends EppResource>
createTld(tld);
contact = persistActiveContact("jd1234");
domain = new DomainResource.Builder()
.setRepoId("1-".concat(tld.toUpperCase()))
.setRepoId("1-".concat(Ascii.toUpperCase(tld)))
.setFullyQualifiedDomainName(label + "." + tld)
.setCurrentSponsorClientId("TheRegistrar")
.setCreationClientId("TheRegistrar")
@ -157,7 +158,7 @@ public class DomainTransferFlowTestCase<F extends Flow, R extends EppResource>
.build());
subordinateHost = persistResource(
new HostResource.Builder()
.setRepoId("2-".concat(tld.toUpperCase()))
.setRepoId("2-".concat(Ascii.toUpperCase(tld)))
.setFullyQualifiedHostName("ns1." + label + "." + tld)
.setCurrentSponsorClientId("TheRegistrar")
.setCreationClientId("TheRegistrar")

8
javatests/google/registry/rdap/testdata/rdap_domain.json vendored
View file

@ -1,9 +1,9 @@
{
"status": [
"delete prohibited",
"renew prohibited",
"transfer prohibited",
"update prohibited"
"client delete prohibited",
"client renew prohibited",
"client transfer prohibited",
"server update prohibited"
],
"handle": "%HANDLE%",
"links": [

8
javatests/google/registry/rdap/testdata/rdap_domain_unicode.json vendored
View file

@ -1,9 +1,9 @@
{
"status": [
"delete prohibited",
"renew prohibited",
"transfer prohibited",
"update prohibited"
"client delete prohibited",
"client renew prohibited",
"client transfer prohibited",
"server update prohibited"
],
"unicodeName": "%NAME%",
"handle": "%HANDLE%",

16
javatests/google/registry/rdap/testdata/rdap_multiple_domains.json vendored
View file

@ -2,10 +2,10 @@
"domainSearchResults": [
{
"status": [
"delete prohibited",
"renew prohibited",
"transfer prohibited",
"update prohibited"
"client delete prohibited",
"client renew prohibited",
"client transfer prohibited",
"server update prohibited"
],
"handle": "21-EXAMPLE",
"links": [
@ -361,10 +361,10 @@
},
{
"status": [
"delete prohibited",
"renew prohibited",
"transfer prohibited",
"update prohibited"
"client delete prohibited",
"client renew prohibited",
"client transfer prohibited",
"server update prohibited"
],
"handle": "C-LOL",
"links": [

8
javatests/google/registry/rdap/testdata/rdapjson_domain_full.json vendored
View file

@ -5,10 +5,10 @@
"unicodeName" : "cat.みんな",
"status" :
[
"delete prohibited",
"renew prohibited",
"transfer prohibited",
"update prohibited"
"client delete prohibited",
"client renew prohibited",
"client transfer prohibited",
"server update prohibited"
],
"links" :
[

8
javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json vendored
View file

@ -5,11 +5,11 @@
"unicodeName" : "fish.みんな",
"status" :
[
"delete prohibited",
"client delete prohibited",
"client renew prohibited",
"client transfer prohibited",
"inactive",
"renew prohibited",
"transfer prohibited",
"update prohibited"
"server update prohibited"
],
"links" :
[

153
javatests/google/registry/rde/RdeImportUtilsTest.java Normal file
View file

@ -0,0 +1,153 @@
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2016 The Domain Registry Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.rde;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.testing.DatastoreHelper.persistResource;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Work;
import google.registry.model.EppResource;
import google.registry.model.contact.ContactResource;
import google.registry.model.index.EppResourceIndex;
import google.registry.model.index.EppResourceIndexBucket;
import google.registry.model.index.ForeignKeyIndex;
import google.registry.testing.AppEngineRule;
import google.registry.testing.FakeClock;
import google.registry.testing.ShardableTestCase;
import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link RdeImportUtils} */
@RunWith(JUnit4.class)
public class RdeImportUtilsTest extends ShardableTestCase {
@Rule
public final AppEngineRule appEngine = AppEngineRule.builder()
.withDatastore()
.build();
private RdeImportUtils rdeImportUtils;
private FakeClock clock;
@Before
public void before() {
clock = new FakeClock();
clock.setTo(DateTime.now());
rdeImportUtils = new RdeImportUtils(ofy(), clock);
}
/** Verifies import of a contact that has not been previously imported */
@Test
public void testImportNewContact() {
ContactResource newContact = buildNewContact();
assertThat(rdeImportUtils.importContact(newContact)).isTrue();
assertEppResourceIndexEntityFor(newContact);
assertForeignKeyIndexFor(newContact);
// verify the new contact was saved
ContactResource saved = getContact("TEST-123");
assertThat(saved).isNotNull();
assertThat(saved.getContactId()).isEqualTo(newContact.getContactId());
assertThat(saved.getEmailAddress()).isEqualTo(newContact.getEmailAddress());
assertThat(saved.getLastEppUpdateTime()).isEqualTo(newContact.getLastEppUpdateTime());
}
/** Verifies that a contact will not be imported more than once */
@Test
public void testImportExistingContact() {
ContactResource newContact = buildNewContact();
persistResource(newContact);
ContactResource updatedContact = newContact.asBuilder()
.setLastEppUpdateTime(newContact.getLastEppUpdateTime().plusSeconds(1))
.build();
assertThat(rdeImportUtils.importContact(updatedContact)).isFalse();
// verify the updated contact was saved
ContactResource saved = getContact("TEST-123");
assertThat(saved).isNotNull();
assertThat(saved.getContactId()).isEqualTo(newContact.getContactId());
assertThat(saved.getEmailAddress()).isEqualTo(newContact.getEmailAddress());
assertThat(saved.getLastEppUpdateTime()).isEqualTo(newContact.getLastEppUpdateTime());
}
private static ContactResource buildNewContact() {
return new ContactResource.Builder()
.setContactId("sh8013")
.setEmailAddress("jdoe@example.com")
.setLastEppUpdateTime(DateTime.parse("2010-10-10T00:00:00.000Z"))
.setRepoId("TEST-123")
.build();
}
private static ContactResource getContact(String repoId) {
final Key<ContactResource> key = Key.create(ContactResource.class, repoId);
return ofy().transact(new Work<ContactResource>() {
@Override
public ContactResource run() {
return ofy().load().key(key).now();
}});
}
/**
* Confirms that a ForeignKeyIndex exists in the datastore for a given resource.
*/
private static <T extends EppResource> void assertForeignKeyIndexFor(final T resource) {
assertThat(ForeignKeyIndex.load(resource.getClass(), resource.getForeignKey(), DateTime.now()))
.isNotNull();
}
/**
* Confirms that an EppResourceIndex entity exists in datastore for a given resource.
*/
private static <T extends EppResource> void assertEppResourceIndexEntityFor(final T resource) {
ImmutableList<EppResourceIndex> indices = FluentIterable
.from(ofy().load()
.type(EppResourceIndex.class)
.filter("kind", Key.getKind(resource.getClass())))
.filter(new Predicate<EppResourceIndex>() {
@Override
public boolean apply(EppResourceIndex index) {
return index.getReference().get().equals(resource);
}})
.toList();
assertThat(indices).hasSize(1);
assertThat(indices.get(0).getBucket())
.isEqualTo(EppResourceIndexBucket.getBucketKey(Key.create(resource)));
}
}

3
javatests/google/registry/rde/RdeReportActionTest.java
View file

@ -38,6 +38,7 @@ import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.ByteSource;
import google.registry.config.RegistryConfig;
@ -194,7 +195,7 @@ public class RdeReportActionTest {
private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
for (HTTPHeader header : headers) {
builder.put(header.getName().replace('-', '_').toUpperCase(), header.getValue());
builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
}
return builder.build();
}

3
javatests/google/registry/testing/DatastoreHelper.java
View file

@ -36,6 +36,7 @@ import static google.registry.util.ResourceUtils.readResourceUtf8;
import static java.util.Arrays.asList;
import static org.joda.money.CurrencyUnit.USD;
import com.google.common.base.Ascii;
import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
@ -360,7 +361,7 @@ public class DatastoreHelper {
}
public static void createTld(String tld, ImmutableSortedMap<DateTime, TldState> tldStates) {
createTld(tld, tld.replaceFirst(ACE_PREFIX_REGEX, "").toUpperCase(), tldStates);
createTld(tld, Ascii.toUpperCase(tld.replaceFirst(ACE_PREFIX_REGEX, "")), tldStates);
}
public static void createTld(

7
javatests/google/registry/testing/TaskQueueHelper.java
View file

@ -30,6 +30,7 @@ import static java.util.Arrays.asList;
import com.google.appengine.api.taskqueue.dev.QueueStateInfo;
import com.google.appengine.api.taskqueue.dev.QueueStateInfo.HeaderWrapper;
import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo;
import com.google.common.base.Ascii;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
@ -94,7 +95,7 @@ public class TaskQueueHelper {
public TaskMatcher header(String name, String value) {
// Lowercase for case-insensitive comparison.
expected.headers.put(name.toLowerCase(), value);
expected.headers.put(Ascii.toLowerCase(name), value);
return this;
}
@ -310,7 +311,7 @@ public class TaskQueueHelper {
for (HeaderWrapper header : info.getHeaders()) {
// Lowercase header name for comparison since HTTP
// header names are case-insensitive.
headerBuilder.put(header.getKey().toLowerCase(), header.getValue());
headerBuilder.put(Ascii.toLowerCase(header.getKey()), header.getValue());
}
this.headers = headerBuilder.build();
ImmutableMultimap.Builder<String, String> inputParams = new ImmutableMultimap.Builder<>();
@ -319,7 +320,7 @@ public class TaskQueueHelper {
inputParams.putAll(UriParameters.parse(query));
}
if (headers.containsEntry(
HttpHeaders.CONTENT_TYPE.toLowerCase(), MediaType.FORM_DATA.toString())) {
Ascii.toLowerCase(HttpHeaders.CONTENT_TYPE), MediaType.FORM_DATA.toString())) {
inputParams.putAll(UriParameters.parse(info.getBody()));
}
this.params = inputParams.build();

4
javatests/google/registry/tools/BUILD
View file

@ -13,7 +13,9 @@ java_library(
srcs = glob([
"*.java",
]),
resources = glob(["testdata/*.*"]),
resources = glob([
"testdata/*.*",
]),
deps = [
"//java/com/google/common/annotations",
"//java/com/google/common/base",

22
javatests/google/registry/tools/CommandTestCase.java
View file

@ -49,7 +49,8 @@ import org.mockito.runners.MockitoJUnitRunner;
@RunWith(MockitoJUnitRunner.class)
public abstract class CommandTestCase<C extends Command> {
private ByteArrayOutputStream stdout = new ByteArrayOutputStream();
private final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
private final ByteArrayOutputStream stderr = new ByteArrayOutputStream();
protected C command;
@ -71,6 +72,7 @@ public abstract class CommandTestCase<C extends Command> {
RegistryToolEnvironment.UNITTEST.setup();
command = newCommandInstance();
System.setOut(new PrintStream(stdout));
System.setErr(new PrintStream(stderr));
}
void runCommandInEnvironment(RegistryToolEnvironment env, String... args) throws Exception {
@ -145,17 +147,29 @@ public abstract class CommandTestCase<C extends Command> {
return ofy().load().type(PollMessage.class).count();
}
protected void assertStdoutIs(String expected) throws Exception {
assertThat(getStdoutAsString()).isEqualTo(expected);
}
protected void assertInStdout(String... expected) throws Exception {
String stdout = getStdoutAsString();
for (String line : expected) {
assertThat(stdout.toString(UTF_8.toString())).contains(line);
assertThat(stdout).contains(line);
}
}
protected void assertInStderr(String... expected) throws Exception {
String stderror = new String(stderr.toByteArray(), UTF_8);
for (String line : expected) {
assertThat(stderror).contains(line);
}
}
void assertNotInStdout(String expected) throws Exception {
assertThat(stdout.toString(UTF_8.toString())).doesNotContain(expected);
assertThat(getStdoutAsString()).doesNotContain(expected);
}
String getStdoutAsString() {
protected String getStdoutAsString() {
return new String(stdout.toByteArray(), UTF_8);
}

18
python/google/registry/reporting/BUILD Normal file
View file

@ -0,0 +1,18 @@
package(default_visibility = ["//java/google/registry:registry_project"])
licenses(["notice"]) # Apache 2.0
py_library(
name = "icann_report_query_builder",
srcs = ["icann_report_query_builder.py"],
deps = ["//python:python_directory_import"],
)
py_test(
name = "icann_report_query_builder_test",
size = "small",
srcs = ["icann_report_query_builder_test.py"],
data = ["testdata/golden_activity_query.sql"],
deps = [":icann_report_query_builder"],
)

392
python/google/registry/reporting/icann_report_query_builder.py Normal file
View file

@ -0,0 +1,392 @@
"""ICANN reporting BigQuery query construction logic.
The IcannReportQueryBuilder class contains logic for constructing the
multi-part BigQuery queries used to produce ICANN monthly reports. These
queries are fairly complicated; see the design doc published to the
domain-registry-users@googlegroups.com for an overview.
Currently, this class only supports building the query for activity
reports (not transaction reports).
"""
import datetime
# This regex pattern matches the full signature of the 'EPP Command' log line
# from FlowRunner.run(), i.e. it matches the logging class/method that prefixes
# the log message, plus the 'EPP Command' string, up to the newline.
# Queries used below depend on matching this log line and parsing its
# exact format, so it must be kept in sync with the logging site.
# TODO(b/20725722): make the log statement format more robust.
FLOWRUNNER_LOG_SIGNATURE_PATTERN = '(?:{}): EPP Command'.format('|'.join([
'com.google.domain.registry.flows.FlowRunner run',
# TODO(b/29397966): figure out why this is FormattingLogger vs FlowRunner.
'com.google.domain.registry.util.FormattingLogger log',
'google.registry.util.FormattingLogger log']))
class IcannReportQueryBuilder(object):
"""Container for methods to build BigQuery queries for ICANN reporting."""
def BuildActivityReportQuery(self, month, registrar_count):
"""Returns the assembled activity report query for a given month.
Specifically, we instantiate the outermost activity report query by pointing
it at the union of a series of "data source" queries that each produce data
used to generate certain metrics. These queries in turn rely on some common
lower-level data source queries (monthly logs, both raw and EPP-parsed).
Args:
month: (str) month of the report to generate, in YYYY-MM format
registrar_count: (int) total number of registrars in the registry system
Returns:
(str) the fully-instantiated activity report query SQL
"""
# Construct some date-related parameters from the given month.
this_month_date = datetime.datetime.strptime(month, '%Y-%m').date()
# Hacky way to compute the start of the next month - add enough days to get
# to the next month (e.g. 31), then set the day to 1. It'd be cleaner to
# use dateutils.relativedelta(months=1) but the dependency is a pain.
month_delta = datetime.timedelta(days=31)
next_month_date = (this_month_date + month_delta).replace(day=1)
this_yearmonth = this_month_date.strftime('%Y-%m')
next_yearmonth = next_month_date.strftime('%Y-%m')
# Construct the queries themselves.
logs_query = self._MakeMonthlyLogsQuery(this_yearmonth, next_yearmonth)
epp_xml_logs_query = self._MakeEppXmlLogsQuery(logs_query)
data_source_queries = [
self._MakeActivityOperationalRegistrarsQuery(next_yearmonth),
self._MakeActivityAllRampedUpRegistrarsQuery(next_yearmonth),
self._MakeActivityAllRegistrarsQuery(registrar_count),
self._MakeActivityWhoisQuery(logs_query), self._MakeActivityDnsQuery(),
self._MakeActivityEppSrsMetricsQuery(epp_xml_logs_query)
]
return _StripTrailingWhitespaceFromLines(self._MakeActivityReportQuery(
data_source_queries))
def _MakeMonthlyLogsQuery(self, this_yearmonth, next_yearmonth):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
query = r"""
-- Query AppEngine request logs for the report month.
SELECT
protoPayload.resource AS requestPath,
protoPayload.line.logMessage AS logMessage,
FROM
TABLE_DATE_RANGE_STRICT(
[appengine_logs.appengine_googleapis_com_request_log_],
TIMESTAMP('%(this_yearmonth)s-01'),
-- End timestamp is inclusive, so subtract 1 second from the
-- timestamp representing the start of the next month.
DATE_ADD(TIMESTAMP('%(next_yearmonth)s-01'), -1, 'SECOND'))
"""
return query % {'this_yearmonth': this_yearmonth,
'next_yearmonth': next_yearmonth}
def _MakeEppXmlLogsQuery(self, logs_query):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
# This query relies on regex-parsing the precise format of the 'EPP Command'
# log line from FlowRunner.run(), so it must be kept in sync.
# TODO(b/20725722): make the log statement format more robust.
query = r"""
-- Query EPP request logs and extract the clientId and raw EPP XML.
SELECT
REGEXP_EXTRACT(logMessage, r'^%(log_signature)s\n\t.+\n\t(.+)\n') AS clientId,
REGEXP_EXTRACT(logMessage, r'^%(log_signature)s\n\t.+\n\t.+\n\t.+\n\t((?s).+)$') AS xml,
FROM (
-- BEGIN LOGS QUERY --
%(logs_query)s
-- END LOGS QUERY --
)
WHERE
-- EPP endpoints from the proxy, regtool, and console respectively.
requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
AND REGEXP_MATCH(logMessage, r'^%(log_signature)s')
"""
return query % {'logs_query': logs_query,
'log_signature': FLOWRUNNER_LOG_SIGNATURE_PATTERN}
def _MakeActivityReportQuery(self, data_source_queries):
"""Make the overall activity report query.
Args:
data_source_queries: list of BigQuery SQL strings to use
as source 'tables' for the main query; each of these
queries must output a schema as follows:
STRING tld / STRING metricName / INTEGER count
A null TLD indicates that the metric counts towards
all TLDs.
Returns:
query as a string of BigQuery SQL
"""
query = r"""
SELECT
Tld.tld AS tld,
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
-- Compute ramp-up-registrars as all-ramped-up-registrars
-- minus operational-registrars, with a floor of 0.
GREATEST(0, SUM(
CASE
WHEN metricName = 'operational-registrars' THEN -count
WHEN metricName = 'all-ramped-up-registrars' THEN count
ELSE 0
END)) AS ramp_up_registrars,
-- Compute pre-ramp-up-registrars as all-registrars minus
-- all-ramp-up-registrars, with a floor of 0.
GREATEST(0, SUM(
CASE
WHEN metricName = 'all-ramped-up-registrars' THEN -count
WHEN metricName = 'all-registrars' THEN count
ELSE 0
END)) AS pre_ramp_up_registrars,
-- We don't support ZFA over SFTP, only AXFR.
0 AS zfa_passwords,
SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries,
-- We don't support searchable WHOIS.
0 AS searchable_whois_queries,
-- DNS queries for UDP/TCP are all assumed to be recevied/responded.
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received,
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded,
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received,
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded,
-- SRS metrics.
SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check,
SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create,
SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete,
SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info,
SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew,
SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report,
SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request,
SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve,
SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel,
SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query,
SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject,
SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request,
SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update,
SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check,
SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create,
SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete,
SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info,
SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update,
SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check,
SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create,
SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete,
SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info,
SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve,
SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel,
SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update,
-- Cross join a list of all TLDs against TLD-specific metrics and then
-- filter so that only metrics with that TLD or a NULL TLD are counted
-- towards a given TLD.
FROM (
SELECT tldStr AS tld
FROM [latest_snapshot.Registry]
-- Include all real TLDs that are not in pre-delegation testing.
WHERE tldType = 'REAL'
OMIT RECORD IF SOME(tldStateTransitions.tldState = 'PDT')
) AS Tld
CROSS JOIN (
SELECT
tld, metricName, count
FROM
-- Dummy data source that ensures that all TLDs appear in report,
-- since they'll all have at least 1 joined row that survives.
(SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 AS count),
-- BEGIN JOINED DATA SOURCES --
%(joined_data_sources)s
-- END JOINED DATA SOURCES --
) AS TldMetrics
WHERE Tld.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
GROUP BY tld
ORDER BY tld
"""
# Turn each data source query into a subquery in parentheses, and join
# them together with comments (representing a table union).
joined_data_sources = '\n' + ',\n'.join(
'(\n%s\n)' % query for query in data_source_queries)
return query % {'joined_data_sources': joined_data_sources}
def _MakeActivityOperationalRegistrarsQuery(self, next_yearmonth):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
query = r"""
-- Query for operational-registrars metric.
SELECT
allowedTlds AS tld,
'operational-registrars' AS metricName,
INTEGER(COUNT(__key__.name)) AS count,
FROM [domain-registry:latest_snapshot.Registrar]
WHERE type = 'REAL'
AND creationTime < TIMESTAMP('%(next_yearmonth)s-01')
GROUP BY tld
"""
return query % {'next_yearmonth': next_yearmonth}
def _MakeActivityAllRampedUpRegistrarsQuery(self, next_yearmonth):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
query = r"""
-- Query for all-ramped-up-registrars metric.
SELECT
STRING(NULL) AS tld, -- Applies to all TLDs.
'all-ramped-up-registrars' AS metricName,
-- Sandbox OT&E registrar names can have either '-{1,2,3,4}' or '{,2,3}'
-- as suffixes - strip all of these off to get the "real" name.
INTEGER(EXACT_COUNT_DISTINCT(
REGEXP_EXTRACT(__key__.name, r'(.+?)(?:-?\d)?$'))) AS count,
FROM [domain-registry-sandbox:latest_snapshot.Registrar]
WHERE type = 'OTE'
AND creationTime < TIMESTAMP('%(next_yearmonth)s-01')
"""
return query % {'next_yearmonth': next_yearmonth}
def _MakeActivityAllRegistrarsQuery(self, registrar_count):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
query = """
-- Query for all-registrars metric.
SELECT
STRING(NULL) AS tld, -- Applies to all TLDs.
'all-registrars' AS metricName,
INTEGER('%(registrar_count)s') AS count,
"""
return query % {'registrar_count': registrar_count}
def _MakeActivityWhoisQuery(self, logs_query):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
query = r"""
-- Query for WHOIS metrics.
SELECT
STRING(NULL) AS tld, -- Applies to all TLDs.
-- Whois queries over port 43 get forwarded by the proxy to /_dr/whois,
-- while web queries come in via /whois/<params>.
CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries'
WHEN LEFT(requestPath, 7) = '/whois/' THEN 'web-whois-queries'
END AS metricName,
INTEGER(COUNT(requestPath)) AS count,
FROM (
-- BEGIN LOGS QUERY --
%(logs_query)s
-- END LOGS QUERY --
)
GROUP BY metricName
HAVING metricName IS NOT NULL
"""
return query % {'logs_query': logs_query}
def _MakeActivityDnsQuery(self):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
query = r"""
-- Query for DNS metrics.
SELECT
STRING(NULL) AS tld,
metricName,
-1 AS count,
FROM
(SELECT 'dns-udp-queries' AS metricName),
(SELECT 'dns-tcp-queries' AS metricName)
"""
return query
def _MakeActivityEppSrsMetricsQuery(self, epp_xml_logs_query):
# TODO(b/20725722): add a real docstring.
# pylint: disable=missing-docstring
query = r"""
-- Query EPP XML messages and calculate SRS metrics.
SELECT
domainTld AS tld,
-- SRS metric names follow a set pattern corresponding to the EPP
-- protocol elements. First we extract the 'inner' command element in
-- EPP, e.g. <domain:create>, which is the resource type followed by
-- the standard EPP command type. To get the metric name, we add the
-- prefix 'srs-', abbreviate 'domain' as 'dom' and 'contact' as 'cont',
-- and replace ':' with '-' to produce 'srs-dom-create'.
--
-- Transfers have subcommands indicated by an 'op' attribute, which we
-- extract and add as an extra suffix for transfer commands, so e.g.
-- 'srs-cont-transfer-approve'. Domain restores are domain updates
-- with a special <rgp:restore> element; if present, the command counts
-- under the srs-dom-rgp-restore-{request,report} metric (depending on
-- the value of the 'op' attribute) instead of srs-dom-update.
CONCAT(
'srs-',
REPLACE(REPLACE(REPLACE(
CASE
WHEN NOT restoreOp IS NULL THEN CONCAT('domain-rgp-restore-', restoreOp)
WHEN commandType = 'transfer' THEN CONCAT(innerCommand, '-', commandOpArg)
ELSE innerCommand
END,
':', '-'), 'domain', 'dom'), 'contact', 'cont')
) AS metricName,
INTEGER(COUNT(xml)) AS count,
FROM (
SELECT
-- Extract salient bits of the EPP XML using regexes. This is fairly
-- safe since the EPP gets schema-validated and pretty-printed before
-- getting logged, and so it looks something like this:
--
-- <command>
-- <transfer op="request">
-- <domain:transfer ...
--
-- From that, we parse out 'transfer' as the command type from the name
-- of the first element after <command>, 'request' as the value of the
-- 'op' attribute of that element (if any), and 'domain:transfer' as
-- the inner command from the name of the subsequent element.
--
-- Domain commands all have at least one <domain:name> element (more
-- than one for domain checks, but we just count the first), from which
-- we extract the domain TLD as everything after the first dot in the
-- element value. This won't work if the client mistakenly sends a
-- hostname (e.g. 'www.foo.example') as the domain name, but we prefer
-- this over taking everything after the last dot so that multipart
-- TLDs like 'co.uk' can be supported.
--
-- Domain restores are indicated by an <rgp:restore> element, from
-- which we extract the value of the 'op' attribute.
--
-- TODO(b/20725722): preprocess the XML in FlowRunner so we don't need
-- regex parsing of XML here (http://stackoverflow.com/a/1732454).
--
REGEXP_EXTRACT(xml, '(?s)<command>.*?<([a-z]+)') AS commandType,
REGEXP_EXTRACT(xml, '(?s)<command>.*?<[a-z]+ op="(.+?)"') AS commandOpArg,
REGEXP_EXTRACT(xml, '(?s)<command>.*?<.+?>.*?<([a-z]+:[a-z]+)') AS innerCommand,
REGEXP_EXTRACT(xml, '<domain:name.*?>[^.]+[.](.+)</domain:name>') AS domainTld,
REGEXP_EXTRACT(xml, '<rgp:restore op="(.+?)"/>') AS restoreOp,
xml,
FROM (
-- BEGIN EPP XML LOGS QUERY --
%(epp_xml_logs_query)s
-- END EPP XML LOGS QUERY --
)
-- Filter to just XML that contains a <command> element (no <hello>s).
WHERE xml CONTAINS '<command>'
)
-- Whitelist of EPP command types that we care about for metrics;
-- excludes login, logout, and poll.
WHERE commandType IN ('check', 'create', 'delete', 'info', 'renew', 'transfer', 'update')
GROUP BY tld, metricName
"""
return query % {'epp_xml_logs_query': epp_xml_logs_query}
def _StripTrailingWhitespaceFromLines(string):
"""Strips trailing whitespace from each line of the provided string.
Args:
string: (str) string to remove trailing whitespace from
Returns:
(str) input string, with trailing whitespace stripped from each line
"""
return '\n'.join(line.rstrip() for line in string.split('\n'))

44
python/google/registry/reporting/icann_report_query_builder_test.py Normal file
View file

@ -0,0 +1,44 @@
"""Tests for google.registry.reporting.icann_report_query_builder."""
import os
import unittest
from google.registry.reporting import icann_report_query_builder
class IcannReportQueryBuilderTest(unittest.TestCase):
testdata_path = None
def setUp(self):
# Using __file__ is a bit of a hack, but it's the only way that "just works"
# for internal and external versions of the code, and it's fine for tests.
self.testdata_path = os.path.join(os.path.dirname(__file__), 'testdata')
def testActivityQuery_matchesGoldenQuery(self):
self.maxDiff = None # Show long diffs
query_builder = icann_report_query_builder.IcannReportQueryBuilder()
golden_activity_query_path = os.path.join(self.testdata_path,
'golden_activity_query.sql')
with open(golden_activity_query_path, 'r') as golden_activity_query:
self.assertMultiLineEqual(golden_activity_query.read(),
query_builder.BuildActivityReportQuery(
month='2016-06',
registrar_count=None))
def testStringTrailingWhitespaceFromLines(self):
def do_test(expected, original):
self.assertEqual(
expected,
icann_report_query_builder._StripTrailingWhitespaceFromLines(
original))
do_test('foo\nbar\nbaz\n', 'foo\nbar\nbaz\n')
do_test('foo\nbar\nbaz\n', 'foo \nbar \nbaz \n')
do_test('foo\nbar\nbaz', 'foo \nbar \nbaz ')
do_test('\nfoo\nbar\nbaz', '\nfoo\nbar\nbaz')
do_test('foo\n\n', 'foo\n \n')
do_test('foo\n', 'foo\n ')
if __name__ == '__main__':
unittest.main()

269
python/google/registry/reporting/testdata/golden_activity_query.sql vendored Normal file
View file

@ -0,0 +1,269 @@
SELECT
Tld.tld AS tld,
SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars,
-- Compute ramp-up-registrars as all-ramped-up-registrars
-- minus operational-registrars, with a floor of 0.
GREATEST(0, SUM(
CASE
WHEN metricName = 'operational-registrars' THEN -count
WHEN metricName = 'all-ramped-up-registrars' THEN count
ELSE 0
END)) AS ramp_up_registrars,
-- Compute pre-ramp-up-registrars as all-registrars minus
-- all-ramp-up-registrars, with a floor of 0.
GREATEST(0, SUM(
CASE
WHEN metricName = 'all-ramped-up-registrars' THEN -count
WHEN metricName = 'all-registrars' THEN count
ELSE 0
END)) AS pre_ramp_up_registrars,
-- We don't support ZFA over SFTP, only AXFR.
0 AS zfa_passwords,
SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries,
SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries,
-- We don't support searchable WHOIS.
0 AS searchable_whois_queries,
-- DNS queries for UDP/TCP are all assumed to be recevied/responded.
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received,
SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded,
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received,
SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded,
-- SRS metrics.
SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check,
SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create,
SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete,
SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info,
SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew,
SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report,
SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request,
SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve,
SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel,
SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query,
SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject,
SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request,
SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update,
SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check,
SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create,
SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete,
SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info,
SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update,
SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check,
SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create,
SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete,
SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info,
SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve,
SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel,
SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query,
SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject,
SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request,
SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update,
-- Cross join a list of all TLDs against TLD-specific metrics and then
-- filter so that only metrics with that TLD or a NULL TLD are counted
-- towards a given TLD.
FROM (
SELECT tldStr AS tld
FROM [latest_snapshot.Registry]
-- Include all real TLDs that are not in pre-delegation testing.
WHERE tldType = 'REAL'
OMIT RECORD IF SOME(tldStateTransitions.tldState = 'PDT')
) AS Tld
CROSS JOIN (
SELECT
tld, metricName, count
FROM
-- Dummy data source that ensures that all TLDs appear in report,
-- since they'll all have at least 1 joined row that survives.
(SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 AS count),
-- BEGIN JOINED DATA SOURCES --
(
-- Query for operational-registrars metric.
SELECT
allowedTlds AS tld,
'operational-registrars' AS metricName,
INTEGER(COUNT(__key__.name)) AS count,
FROM [domain-registry:latest_snapshot.Registrar]
WHERE type = 'REAL'
AND creationTime < TIMESTAMP('2016-07-01')
GROUP BY tld
),
(
-- Query for all-ramped-up-registrars metric.
SELECT
STRING(NULL) AS tld, -- Applies to all TLDs.
'all-ramped-up-registrars' AS metricName,
-- Sandbox OT&E registrar names can have either '-{1,2,3,4}' or '{,2,3}'
-- as suffixes - strip all of these off to get the "real" name.
INTEGER(EXACT_COUNT_DISTINCT(
REGEXP_EXTRACT(__key__.name, r'(.+?)(?:-?\d)?$'))) AS count,
FROM [domain-registry-sandbox:latest_snapshot.Registrar]
WHERE type = 'OTE'
AND creationTime < TIMESTAMP('2016-07-01')
),
(
-- Query for all-registrars metric.
SELECT
STRING(NULL) AS tld, -- Applies to all TLDs.
'all-registrars' AS metricName,
INTEGER('None') AS count,
),
(
-- Query for WHOIS metrics.
SELECT
STRING(NULL) AS tld, -- Applies to all TLDs.
-- Whois queries over port 43 get forwarded by the proxy to /_dr/whois,
-- while web queries come in via /whois/<params>.
CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries'
WHEN LEFT(requestPath, 7) = '/whois/' THEN 'web-whois-queries'
END AS metricName,
INTEGER(COUNT(requestPath)) AS count,
FROM (
-- BEGIN LOGS QUERY --
-- Query AppEngine request logs for the report month.
SELECT
protoPayload.resource AS requestPath,
protoPayload.line.logMessage AS logMessage,
FROM
TABLE_DATE_RANGE_STRICT(
[appengine_logs.appengine_googleapis_com_request_log_],
TIMESTAMP('2016-06-01'),
-- End timestamp is inclusive, so subtract 1 second from the
-- timestamp representing the start of the next month.
DATE_ADD(TIMESTAMP('2016-07-01'), -1, 'SECOND'))
-- END LOGS QUERY --
)
GROUP BY metricName
HAVING metricName IS NOT NULL
),
(
-- Query for DNS metrics.
SELECT
STRING(NULL) AS tld,
metricName,
-1 AS count,
FROM
(SELECT 'dns-udp-queries' AS metricName),
(SELECT 'dns-tcp-queries' AS metricName)
),
(
-- Query EPP XML messages and calculate SRS metrics.
SELECT
domainTld AS tld,
-- SRS metric names follow a set pattern corresponding to the EPP
-- protocol elements. First we extract the 'inner' command element in
-- EPP, e.g. <domain:create>, which is the resource type followed by
-- the standard EPP command type. To get the metric name, we add the
-- prefix 'srs-', abbreviate 'domain' as 'dom' and 'contact' as 'cont',
-- and replace ':' with '-' to produce 'srs-dom-create'.
--
-- Transfers have subcommands indicated by an 'op' attribute, which we
-- extract and add as an extra suffix for transfer commands, so e.g.
-- 'srs-cont-transfer-approve'. Domain restores are domain updates
-- with a special <rgp:restore> element; if present, the command counts
-- under the srs-dom-rgp-restore-{request,report} metric (depending on
-- the value of the 'op' attribute) instead of srs-dom-update.
CONCAT(
'srs-',
REPLACE(REPLACE(REPLACE(
CASE
WHEN NOT restoreOp IS NULL THEN CONCAT('domain-rgp-restore-', restoreOp)
WHEN commandType = 'transfer' THEN CONCAT(innerCommand, '-', commandOpArg)
ELSE innerCommand
END,
':', '-'), 'domain', 'dom'), 'contact', 'cont')
) AS metricName,
INTEGER(COUNT(xml)) AS count,
FROM (
SELECT
-- Extract salient bits of the EPP XML using regexes. This is fairly
-- safe since the EPP gets schema-validated and pretty-printed before
-- getting logged, and so it looks something like this:
--
-- <command>
-- <transfer op="request">
-- <domain:transfer ...
--
-- From that, we parse out 'transfer' as the command type from the name
-- of the first element after <command>, 'request' as the value of the
-- 'op' attribute of that element (if any), and 'domain:transfer' as
-- the inner command from the name of the subsequent element.
--
-- Domain commands all have at least one <domain:name> element (more
-- than one for domain checks, but we just count the first), from which
-- we extract the domain TLD as everything after the first dot in the
-- element value. This won't work if the client mistakenly sends a
-- hostname (e.g. 'www.foo.example') as the domain name, but we prefer
-- this over taking everything after the last dot so that multipart
-- TLDs like 'co.uk' can be supported.
--
-- Domain restores are indicated by an <rgp:restore> element, from
-- which we extract the value of the 'op' attribute.
--
-- TODO(b/20725722): preprocess the XML in FlowRunner so we don't need
-- regex parsing of XML here (http://stackoverflow.com/a/1732454).
--
REGEXP_EXTRACT(xml, '(?s)<command>.*?<([a-z]+)') AS commandType,
REGEXP_EXTRACT(xml, '(?s)<command>.*?<[a-z]+ op="(.+?)"') AS commandOpArg,
REGEXP_EXTRACT(xml, '(?s)<command>.*?<.+?>.*?<([a-z]+:[a-z]+)') AS innerCommand,
REGEXP_EXTRACT(xml, '<domain:name.*?>[^.]+[.](.+)</domain:name>') AS domainTld,
REGEXP_EXTRACT(xml, '<rgp:restore op="(.+?)"/>') AS restoreOp,
xml,
FROM (
-- BEGIN EPP XML LOGS QUERY --
-- Query EPP request logs and extract the clientId and raw EPP XML.
SELECT
REGEXP_EXTRACT(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command\n\t.+\n\t(.+)\n') AS clientId,
REGEXP_EXTRACT(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command\n\t.+\n\t.+\n\t.+\n\t((?s).+)$') AS xml,
FROM (
-- BEGIN LOGS QUERY --
-- Query AppEngine request logs for the report month.
SELECT
protoPayload.resource AS requestPath,
protoPayload.line.logMessage AS logMessage,
FROM
TABLE_DATE_RANGE_STRICT(
[appengine_logs.appengine_googleapis_com_request_log_],
TIMESTAMP('2016-06-01'),
-- End timestamp is inclusive, so subtract 1 second from the
-- timestamp representing the start of the next month.
DATE_ADD(TIMESTAMP('2016-07-01'), -1, 'SECOND'))
-- END LOGS QUERY --
)
WHERE
-- EPP endpoints from the proxy, regtool, and console respectively.
requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
AND REGEXP_MATCH(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command')
-- END EPP XML LOGS QUERY --
)
-- Filter to just XML that contains a <command> element (no <hello>s).
WHERE xml CONTAINS '<command>'
)
-- Whitelist of EPP command types that we care about for metrics;
-- excludes login, logout, and poll.
WHERE commandType IN ('check', 'create', 'delete', 'info', 'renew', 'transfer', 'update')
GROUP BY tld, metricName
)
-- END JOINED DATA SOURCES --
) AS TldMetrics
WHERE Tld.tld = TldMetrics.tld OR TldMetrics.tld IS NULL
GROUP BY tld
ORDER BY tld