From bca9b2a80699728b7de18a3e7208563394e578ae Mon Sep 17 00:00:00 2001 From: mcilwain Date: Fri, 15 Jul 2016 11:25:42 -0700 Subject: [PATCH 1/8] Add stubs for Markdown documentation ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127560220 --- docs/app-engine-architecture.md | 13 +++++++++++++ docs/code-structure.md | 3 +++ docs/configuration.md | 3 +++ docs/developing.md | 3 +++ docs/extension-points.md | 3 +++ docs/install.md | 3 +++ docs/registry-tool.md | 3 +++ 7 files changed, 31 insertions(+) create mode 100644 docs/app-engine-architecture.md create mode 100644 docs/code-structure.md create mode 100644 docs/configuration.md create mode 100644 docs/developing.md create mode 100644 docs/extension-points.md create mode 100644 docs/install.md create mode 100644 docs/registry-tool.md diff --git a/docs/app-engine-architecture.md b/docs/app-engine-architecture.md new file mode 100644 index 000000000..cdbccdc23 --- /dev/null +++ b/docs/app-engine-architecture.md @@ -0,0 +1,13 @@ +# App Engine architecture + +This document contains information on the overall architecture of the Domain Registry project as it is implemented in App Engine. + +## Modules + +## Task queues + +## Cron tasks + +## Datastore entities + +## Cloud Storage buckets diff --git a/docs/code-structure.md b/docs/code-structure.md new file mode 100644 index 000000000..8196c035e --- /dev/null +++ b/docs/code-structure.md @@ -0,0 +1,3 @@ +# Code structure + +An overall look at the structure of the Domain Registry code. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..831d4e792 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,3 @@ +# Configuration + +Information on how to configure/customize an installation. diff --git a/docs/developing.md b/docs/developing.md new file mode 100644 index 000000000..4ac317558 --- /dev/null +++ b/docs/developing.md @@ -0,0 +1,3 @@ +# Developing + +Advice on how to do development on the Domain Registry codebase (including how to set up an IDE environment and run tests). diff --git a/docs/extension-points.md b/docs/extension-points.md new file mode 100644 index 000000000..68f5118b8 --- /dev/null +++ b/docs/extension-points.md @@ -0,0 +1,3 @@ +# Extension points + +The various places the system can be extended by plugging in additional code. diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 000000000..419e75b20 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,3 @@ +# Installation + +Information on how to download and install the Domain Registry project and get a working running instance. diff --git a/docs/registry-tool.md b/docs/registry-tool.md new file mode 100644 index 000000000..d3915bbee --- /dev/null +++ b/docs/registry-tool.md @@ -0,0 +1,3 @@ +# Registry tool + +Information on registry_tool commands. From 60180348cde540f51af49c6a262e950a38feda48 Mon Sep 17 00:00:00 2001 From: mcilwain Date: Fri, 15 Jul 2016 11:38:50 -0700 Subject: [PATCH 2/8] Add better assertions on registry_tool stdout ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127561648 --- javatests/google/registry/tools/CommandTestCase.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/javatests/google/registry/tools/CommandTestCase.java b/javatests/google/registry/tools/CommandTestCase.java index 80ec46f14..5c26f82b2 100644 --- a/javatests/google/registry/tools/CommandTestCase.java +++ b/javatests/google/registry/tools/CommandTestCase.java @@ -145,17 +145,22 @@ public abstract class CommandTestCase { return ofy().load().type(PollMessage.class).count(); } + protected void assertStdoutIs(String expected) throws Exception { + assertThat(getStdoutAsString()).isEqualTo(expected); + } + protected void assertInStdout(String... expected) throws Exception { + String stdout = getStdoutAsString(); for (String line : expected) { - assertThat(stdout.toString(UTF_8.toString())).contains(line); + assertThat(stdout).contains(line); } } void assertNotInStdout(String expected) throws Exception { - assertThat(stdout.toString(UTF_8.toString())).doesNotContain(expected); + assertThat(getStdoutAsString()).doesNotContain(expected); } - String getStdoutAsString() { + protected String getStdoutAsString() { return new String(stdout.toByteArray(), UTF_8); } From 2714e07f06e5309a293fba2c03c13f7a0964149a Mon Sep 17 00:00:00 2001 From: jart Date: Fri, 15 Jul 2016 13:08:56 -0700 Subject: [PATCH 3/8] Follow 80 column rule in docs Markdown ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127570507 --- docs/app-engine-architecture.md | 3 ++- docs/developing.md | 3 ++- docs/install.md | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/app-engine-architecture.md b/docs/app-engine-architecture.md index cdbccdc23..7ca73cf0d 100644 --- a/docs/app-engine-architecture.md +++ b/docs/app-engine-architecture.md @@ -1,6 +1,7 @@ # App Engine architecture -This document contains information on the overall architecture of the Domain Registry project as it is implemented in App Engine. +This document contains information on the overall architecture of the Domain +Registry project as it is implemented in App Engine. ## Modules diff --git a/docs/developing.md b/docs/developing.md index 4ac317558..cb4d13070 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -1,3 +1,4 @@ # Developing -Advice on how to do development on the Domain Registry codebase (including how to set up an IDE environment and run tests). +Advice on how to do development on the Domain Registry codebase (including how +to set up an IDE environment and run tests). diff --git a/docs/install.md b/docs/install.md index 419e75b20..90aaf9486 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,3 +1,4 @@ # Installation -Information on how to download and install the Domain Registry project and get a working running instance. +Information on how to download and install the Domain Registry project and get a +working running instance. From b9d1a4362fc5d0381d686839efe263947f28ca89 Mon Sep 17 00:00:00 2001 From: nickfelt Date: Fri, 15 Jul 2016 14:40:41 -0700 Subject: [PATCH 4/8] Move ICANN activity report BigQuery SQL code to opensource repo This migrates the IcannReportQueryBuilder part of the internal ICANN reporting script into the opensource repository, as a new module under the package google.registry.reporting. It correspondingly moves the golden activity SQL query test to the opensource repo, since that test only applies to this part of the script anyway (note that the actual golden SQL contents is unchanged by the move). Tested: confirmed that the newly moved test passes (and that it also fails when expected as well), and ran the internal icann reporting script locally to verify that both activity and transaction reporting results are unaffected by the move. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127580326 --- python/google/registry/reporting/BUILD | 18 + .../reporting/icann_report_query_builder.py | 392 ++++++++++++++++++ .../icann_report_query_builder_test.py | 44 ++ .../testdata/golden_activity_query.sql | 269 ++++++++++++ 4 files changed, 723 insertions(+) create mode 100644 python/google/registry/reporting/BUILD create mode 100644 python/google/registry/reporting/icann_report_query_builder.py create mode 100644 python/google/registry/reporting/icann_report_query_builder_test.py create mode 100644 python/google/registry/reporting/testdata/golden_activity_query.sql diff --git a/python/google/registry/reporting/BUILD b/python/google/registry/reporting/BUILD new file mode 100644 index 000000000..4f4c7e073 --- /dev/null +++ b/python/google/registry/reporting/BUILD @@ -0,0 +1,18 @@ +package(default_visibility = ["//java/google/registry:registry_project"]) + +licenses(["notice"]) # Apache 2.0 + + +py_library( + name = "icann_report_query_builder", + srcs = ["icann_report_query_builder.py"], + deps = ["//python:python_directory_import"], +) + +py_test( + name = "icann_report_query_builder_test", + size = "small", + srcs = ["icann_report_query_builder_test.py"], + data = ["testdata/golden_activity_query.sql"], + deps = [":icann_report_query_builder"], +) diff --git a/python/google/registry/reporting/icann_report_query_builder.py b/python/google/registry/reporting/icann_report_query_builder.py new file mode 100644 index 000000000..b1f17bd74 --- /dev/null +++ b/python/google/registry/reporting/icann_report_query_builder.py @@ -0,0 +1,392 @@ +"""ICANN reporting BigQuery query construction logic. + +The IcannReportQueryBuilder class contains logic for constructing the +multi-part BigQuery queries used to produce ICANN monthly reports. These +queries are fairly complicated; see the design doc published to the +domain-registry-users@googlegroups.com for an overview. + +Currently, this class only supports building the query for activity +reports (not transaction reports). +""" +import datetime + +# This regex pattern matches the full signature of the 'EPP Command' log line +# from FlowRunner.run(), i.e. it matches the logging class/method that prefixes +# the log message, plus the 'EPP Command' string, up to the newline. +# Queries used below depend on matching this log line and parsing its +# exact format, so it must be kept in sync with the logging site. +# TODO(b/20725722): make the log statement format more robust. +FLOWRUNNER_LOG_SIGNATURE_PATTERN = '(?:{}): EPP Command'.format('|'.join([ + 'com.google.domain.registry.flows.FlowRunner run', + # TODO(b/29397966): figure out why this is FormattingLogger vs FlowRunner. + 'com.google.domain.registry.util.FormattingLogger log', + 'google.registry.util.FormattingLogger log'])) + + +class IcannReportQueryBuilder(object): + """Container for methods to build BigQuery queries for ICANN reporting.""" + + def BuildActivityReportQuery(self, month, registrar_count): + """Returns the assembled activity report query for a given month. + + Specifically, we instantiate the outermost activity report query by pointing + it at the union of a series of "data source" queries that each produce data + used to generate certain metrics. These queries in turn rely on some common + lower-level data source queries (monthly logs, both raw and EPP-parsed). + + Args: + month: (str) month of the report to generate, in YYYY-MM format + registrar_count: (int) total number of registrars in the registry system + + Returns: + (str) the fully-instantiated activity report query SQL + """ + # Construct some date-related parameters from the given month. + this_month_date = datetime.datetime.strptime(month, '%Y-%m').date() + # Hacky way to compute the start of the next month - add enough days to get + # to the next month (e.g. 31), then set the day to 1. It'd be cleaner to + # use dateutils.relativedelta(months=1) but the dependency is a pain. + month_delta = datetime.timedelta(days=31) + next_month_date = (this_month_date + month_delta).replace(day=1) + this_yearmonth = this_month_date.strftime('%Y-%m') + next_yearmonth = next_month_date.strftime('%Y-%m') + + # Construct the queries themselves. + logs_query = self._MakeMonthlyLogsQuery(this_yearmonth, next_yearmonth) + epp_xml_logs_query = self._MakeEppXmlLogsQuery(logs_query) + data_source_queries = [ + self._MakeActivityOperationalRegistrarsQuery(next_yearmonth), + self._MakeActivityAllRampedUpRegistrarsQuery(next_yearmonth), + self._MakeActivityAllRegistrarsQuery(registrar_count), + self._MakeActivityWhoisQuery(logs_query), self._MakeActivityDnsQuery(), + self._MakeActivityEppSrsMetricsQuery(epp_xml_logs_query) + ] + return _StripTrailingWhitespaceFromLines(self._MakeActivityReportQuery( + data_source_queries)) + + def _MakeMonthlyLogsQuery(self, this_yearmonth, next_yearmonth): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query AppEngine request logs for the report month. + SELECT + protoPayload.resource AS requestPath, + protoPayload.line.logMessage AS logMessage, + FROM + TABLE_DATE_RANGE_STRICT( + [appengine_logs.appengine_googleapis_com_request_log_], + TIMESTAMP('%(this_yearmonth)s-01'), + -- End timestamp is inclusive, so subtract 1 second from the + -- timestamp representing the start of the next month. + DATE_ADD(TIMESTAMP('%(next_yearmonth)s-01'), -1, 'SECOND')) + """ + return query % {'this_yearmonth': this_yearmonth, + 'next_yearmonth': next_yearmonth} + + def _MakeEppXmlLogsQuery(self, logs_query): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + # This query relies on regex-parsing the precise format of the 'EPP Command' + # log line from FlowRunner.run(), so it must be kept in sync. + # TODO(b/20725722): make the log statement format more robust. + query = r""" + -- Query EPP request logs and extract the clientId and raw EPP XML. + SELECT + REGEXP_EXTRACT(logMessage, r'^%(log_signature)s\n\t.+\n\t(.+)\n') AS clientId, + REGEXP_EXTRACT(logMessage, r'^%(log_signature)s\n\t.+\n\t.+\n\t.+\n\t((?s).+)$') AS xml, + FROM ( + -- BEGIN LOGS QUERY -- + %(logs_query)s + -- END LOGS QUERY -- + ) + WHERE + -- EPP endpoints from the proxy, regtool, and console respectively. + requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr') + AND REGEXP_MATCH(logMessage, r'^%(log_signature)s') + """ + return query % {'logs_query': logs_query, + 'log_signature': FLOWRUNNER_LOG_SIGNATURE_PATTERN} + + def _MakeActivityReportQuery(self, data_source_queries): + """Make the overall activity report query. + + Args: + data_source_queries: list of BigQuery SQL strings to use + as source 'tables' for the main query; each of these + queries must output a schema as follows: + + STRING tld / STRING metricName / INTEGER count + + A null TLD indicates that the metric counts towards + all TLDs. + + Returns: + query as a string of BigQuery SQL + """ + query = r""" + SELECT + Tld.tld AS tld, + SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars, + -- Compute ramp-up-registrars as all-ramped-up-registrars + -- minus operational-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'operational-registrars' THEN -count + WHEN metricName = 'all-ramped-up-registrars' THEN count + ELSE 0 + END)) AS ramp_up_registrars, + -- Compute pre-ramp-up-registrars as all-registrars minus + -- all-ramp-up-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'all-ramped-up-registrars' THEN -count + WHEN metricName = 'all-registrars' THEN count + ELSE 0 + END)) AS pre_ramp_up_registrars, + -- We don't support ZFA over SFTP, only AXFR. + 0 AS zfa_passwords, + SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries, + SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries, + -- We don't support searchable WHOIS. + 0 AS searchable_whois_queries, + -- DNS queries for UDP/TCP are all assumed to be recevied/responded. + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received, + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded, + -- SRS metrics. + SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check, + SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create, + SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete, + SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info, + SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew, + SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report, + SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request, + SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve, + SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel, + SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query, + SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject, + SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request, + SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update, + SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check, + SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create, + SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete, + SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info, + SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update, + SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check, + SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create, + SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete, + SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info, + SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve, + SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel, + SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query, + SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject, + SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request, + SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update, + -- Cross join a list of all TLDs against TLD-specific metrics and then + -- filter so that only metrics with that TLD or a NULL TLD are counted + -- towards a given TLD. + FROM ( + SELECT tldStr AS tld + FROM [latest_snapshot.Registry] + -- Include all real TLDs that are not in pre-delegation testing. + WHERE tldType = 'REAL' + OMIT RECORD IF SOME(tldStateTransitions.tldState = 'PDT') + ) AS Tld + CROSS JOIN ( + SELECT + tld, metricName, count + FROM + -- Dummy data source that ensures that all TLDs appear in report, + -- since they'll all have at least 1 joined row that survives. + (SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 AS count), + -- BEGIN JOINED DATA SOURCES -- + %(joined_data_sources)s + -- END JOINED DATA SOURCES -- + ) AS TldMetrics + WHERE Tld.tld = TldMetrics.tld OR TldMetrics.tld IS NULL + GROUP BY tld + ORDER BY tld + """ + # Turn each data source query into a subquery in parentheses, and join + # them together with comments (representing a table union). + joined_data_sources = '\n' + ',\n'.join( + '(\n%s\n)' % query for query in data_source_queries) + return query % {'joined_data_sources': joined_data_sources} + + def _MakeActivityOperationalRegistrarsQuery(self, next_yearmonth): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for operational-registrars metric. + SELECT + allowedTlds AS tld, + 'operational-registrars' AS metricName, + INTEGER(COUNT(__key__.name)) AS count, + FROM [domain-registry:latest_snapshot.Registrar] + WHERE type = 'REAL' + AND creationTime < TIMESTAMP('%(next_yearmonth)s-01') + GROUP BY tld + """ + return query % {'next_yearmonth': next_yearmonth} + + def _MakeActivityAllRampedUpRegistrarsQuery(self, next_yearmonth): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for all-ramped-up-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-ramped-up-registrars' AS metricName, + -- Sandbox OT&E registrar names can have either '-{1,2,3,4}' or '{,2,3}' + -- as suffixes - strip all of these off to get the "real" name. + INTEGER(EXACT_COUNT_DISTINCT( + REGEXP_EXTRACT(__key__.name, r'(.+?)(?:-?\d)?$'))) AS count, + FROM [domain-registry-sandbox:latest_snapshot.Registrar] + WHERE type = 'OTE' + AND creationTime < TIMESTAMP('%(next_yearmonth)s-01') + """ + return query % {'next_yearmonth': next_yearmonth} + + def _MakeActivityAllRegistrarsQuery(self, registrar_count): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = """ + -- Query for all-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-registrars' AS metricName, + INTEGER('%(registrar_count)s') AS count, + """ + return query % {'registrar_count': registrar_count} + + def _MakeActivityWhoisQuery(self, logs_query): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for WHOIS metrics. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + -- Whois queries over port 43 get forwarded by the proxy to /_dr/whois, + -- while web queries come in via /whois/. + CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries' + WHEN LEFT(requestPath, 7) = '/whois/' THEN 'web-whois-queries' + END AS metricName, + INTEGER(COUNT(requestPath)) AS count, + FROM ( + -- BEGIN LOGS QUERY -- + %(logs_query)s + -- END LOGS QUERY -- + ) + GROUP BY metricName + HAVING metricName IS NOT NULL + """ + return query % {'logs_query': logs_query} + + def _MakeActivityDnsQuery(self): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query for DNS metrics. + SELECT + STRING(NULL) AS tld, + metricName, + -1 AS count, + FROM + (SELECT 'dns-udp-queries' AS metricName), + (SELECT 'dns-tcp-queries' AS metricName) + """ + return query + + def _MakeActivityEppSrsMetricsQuery(self, epp_xml_logs_query): + # TODO(b/20725722): add a real docstring. + # pylint: disable=missing-docstring + query = r""" + -- Query EPP XML messages and calculate SRS metrics. + SELECT + domainTld AS tld, + -- SRS metric names follow a set pattern corresponding to the EPP + -- protocol elements. First we extract the 'inner' command element in + -- EPP, e.g. , which is the resource type followed by + -- the standard EPP command type. To get the metric name, we add the + -- prefix 'srs-', abbreviate 'domain' as 'dom' and 'contact' as 'cont', + -- and replace ':' with '-' to produce 'srs-dom-create'. + -- + -- Transfers have subcommands indicated by an 'op' attribute, which we + -- extract and add as an extra suffix for transfer commands, so e.g. + -- 'srs-cont-transfer-approve'. Domain restores are domain updates + -- with a special element; if present, the command counts + -- under the srs-dom-rgp-restore-{request,report} metric (depending on + -- the value of the 'op' attribute) instead of srs-dom-update. + CONCAT( + 'srs-', + REPLACE(REPLACE(REPLACE( + CASE + WHEN NOT restoreOp IS NULL THEN CONCAT('domain-rgp-restore-', restoreOp) + WHEN commandType = 'transfer' THEN CONCAT(innerCommand, '-', commandOpArg) + ELSE innerCommand + END, + ':', '-'), 'domain', 'dom'), 'contact', 'cont') + ) AS metricName, + INTEGER(COUNT(xml)) AS count, + FROM ( + SELECT + -- Extract salient bits of the EPP XML using regexes. This is fairly + -- safe since the EPP gets schema-validated and pretty-printed before + -- getting logged, and so it looks something like this: + -- + -- + -- + -- , 'request' as the value of the + -- 'op' attribute of that element (if any), and 'domain:transfer' as + -- the inner command from the name of the subsequent element. + -- + -- Domain commands all have at least one element (more + -- than one for domain checks, but we just count the first), from which + -- we extract the domain TLD as everything after the first dot in the + -- element value. This won't work if the client mistakenly sends a + -- hostname (e.g. 'www.foo.example') as the domain name, but we prefer + -- this over taking everything after the last dot so that multipart + -- TLDs like 'co.uk' can be supported. + -- + -- Domain restores are indicated by an element, from + -- which we extract the value of the 'op' attribute. + -- + -- TODO(b/20725722): preprocess the XML in FlowRunner so we don't need + -- regex parsing of XML here (http://stackoverflow.com/a/1732454). + -- + REGEXP_EXTRACT(xml, '(?s).*?<([a-z]+)') AS commandType, + REGEXP_EXTRACT(xml, '(?s).*?<[a-z]+ op="(.+?)"') AS commandOpArg, + REGEXP_EXTRACT(xml, '(?s).*?<.+?>.*?<([a-z]+:[a-z]+)') AS innerCommand, + REGEXP_EXTRACT(xml, '[^.]+[.](.+)') AS domainTld, + REGEXP_EXTRACT(xml, '') AS restoreOp, + xml, + FROM ( + -- BEGIN EPP XML LOGS QUERY -- + %(epp_xml_logs_query)s + -- END EPP XML LOGS QUERY -- + ) + -- Filter to just XML that contains a element (no s). + WHERE xml CONTAINS '' + ) + -- Whitelist of EPP command types that we care about for metrics; + -- excludes login, logout, and poll. + WHERE commandType IN ('check', 'create', 'delete', 'info', 'renew', 'transfer', 'update') + GROUP BY tld, metricName + """ + return query % {'epp_xml_logs_query': epp_xml_logs_query} + + +def _StripTrailingWhitespaceFromLines(string): + """Strips trailing whitespace from each line of the provided string. + + Args: + string: (str) string to remove trailing whitespace from + + Returns: + (str) input string, with trailing whitespace stripped from each line + """ + return '\n'.join(line.rstrip() for line in string.split('\n')) diff --git a/python/google/registry/reporting/icann_report_query_builder_test.py b/python/google/registry/reporting/icann_report_query_builder_test.py new file mode 100644 index 000000000..32c595d50 --- /dev/null +++ b/python/google/registry/reporting/icann_report_query_builder_test.py @@ -0,0 +1,44 @@ +"""Tests for google.registry.reporting.icann_report_query_builder.""" + +import os +import unittest + +from google.registry.reporting import icann_report_query_builder + + +class IcannReportQueryBuilderTest(unittest.TestCase): + + testdata_path = None + + def setUp(self): + # Using __file__ is a bit of a hack, but it's the only way that "just works" + # for internal and external versions of the code, and it's fine for tests. + self.testdata_path = os.path.join(os.path.dirname(__file__), 'testdata') + + def testActivityQuery_matchesGoldenQuery(self): + self.maxDiff = None # Show long diffs + query_builder = icann_report_query_builder.IcannReportQueryBuilder() + golden_activity_query_path = os.path.join(self.testdata_path, + 'golden_activity_query.sql') + with open(golden_activity_query_path, 'r') as golden_activity_query: + self.assertMultiLineEqual(golden_activity_query.read(), + query_builder.BuildActivityReportQuery( + month='2016-06', + registrar_count=None)) + + def testStringTrailingWhitespaceFromLines(self): + def do_test(expected, original): + self.assertEqual( + expected, + icann_report_query_builder._StripTrailingWhitespaceFromLines( + original)) + do_test('foo\nbar\nbaz\n', 'foo\nbar\nbaz\n') + do_test('foo\nbar\nbaz\n', 'foo \nbar \nbaz \n') + do_test('foo\nbar\nbaz', 'foo \nbar \nbaz ') + do_test('\nfoo\nbar\nbaz', '\nfoo\nbar\nbaz') + do_test('foo\n\n', 'foo\n \n') + do_test('foo\n', 'foo\n ') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/google/registry/reporting/testdata/golden_activity_query.sql b/python/google/registry/reporting/testdata/golden_activity_query.sql new file mode 100644 index 000000000..4e7318dc8 --- /dev/null +++ b/python/google/registry/reporting/testdata/golden_activity_query.sql @@ -0,0 +1,269 @@ + + SELECT + Tld.tld AS tld, + SUM(IF(metricName = 'operational-registrars', count, 0)) AS operational_registrars, + -- Compute ramp-up-registrars as all-ramped-up-registrars + -- minus operational-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'operational-registrars' THEN -count + WHEN metricName = 'all-ramped-up-registrars' THEN count + ELSE 0 + END)) AS ramp_up_registrars, + -- Compute pre-ramp-up-registrars as all-registrars minus + -- all-ramp-up-registrars, with a floor of 0. + GREATEST(0, SUM( + CASE + WHEN metricName = 'all-ramped-up-registrars' THEN -count + WHEN metricName = 'all-registrars' THEN count + ELSE 0 + END)) AS pre_ramp_up_registrars, + -- We don't support ZFA over SFTP, only AXFR. + 0 AS zfa_passwords, + SUM(IF(metricName = 'whois-43-queries', count, 0)) AS whois_43_queries, + SUM(IF(metricName = 'web-whois-queries', count, 0)) AS web_whois_queries, + -- We don't support searchable WHOIS. + 0 AS searchable_whois_queries, + -- DNS queries for UDP/TCP are all assumed to be recevied/responded. + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_received, + SUM(IF(metricName = 'dns-udp-queries', count, 0)) AS dns_udp_queries_responded, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_received, + SUM(IF(metricName = 'dns-tcp-queries', count, 0)) AS dns_tcp_queries_responded, + -- SRS metrics. + SUM(IF(metricName = 'srs-dom-check', count, 0)) AS srs_dom_check, + SUM(IF(metricName = 'srs-dom-create', count, 0)) AS srs_dom_create, + SUM(IF(metricName = 'srs-dom-delete', count, 0)) AS srs_dom_delete, + SUM(IF(metricName = 'srs-dom-info', count, 0)) AS srs_dom_info, + SUM(IF(metricName = 'srs-dom-renew', count, 0)) AS srs_dom_renew, + SUM(IF(metricName = 'srs-dom-rgp-restore-report', count, 0)) AS srs_dom_rgp_restore_report, + SUM(IF(metricName = 'srs-dom-rgp-restore-request', count, 0)) AS srs_dom_rgp_restore_request, + SUM(IF(metricName = 'srs-dom-transfer-approve', count, 0)) AS srs_dom_transfer_approve, + SUM(IF(metricName = 'srs-dom-transfer-cancel', count, 0)) AS srs_dom_transfer_cancel, + SUM(IF(metricName = 'srs-dom-transfer-query', count, 0)) AS srs_dom_transfer_query, + SUM(IF(metricName = 'srs-dom-transfer-reject', count, 0)) AS srs_dom_transfer_reject, + SUM(IF(metricName = 'srs-dom-transfer-request', count, 0)) AS srs_dom_transfer_request, + SUM(IF(metricName = 'srs-dom-update', count, 0)) AS srs_dom_update, + SUM(IF(metricName = 'srs-host-check', count, 0)) AS srs_host_check, + SUM(IF(metricName = 'srs-host-create', count, 0)) AS srs_host_create, + SUM(IF(metricName = 'srs-host-delete', count, 0)) AS srs_host_delete, + SUM(IF(metricName = 'srs-host-info', count, 0)) AS srs_host_info, + SUM(IF(metricName = 'srs-host-update', count, 0)) AS srs_host_update, + SUM(IF(metricName = 'srs-cont-check', count, 0)) AS srs_cont_check, + SUM(IF(metricName = 'srs-cont-create', count, 0)) AS srs_cont_create, + SUM(IF(metricName = 'srs-cont-delete', count, 0)) AS srs_cont_delete, + SUM(IF(metricName = 'srs-cont-info', count, 0)) AS srs_cont_info, + SUM(IF(metricName = 'srs-cont-transfer-approve', count, 0)) AS srs_cont_transfer_approve, + SUM(IF(metricName = 'srs-cont-transfer-cancel', count, 0)) AS srs_cont_transfer_cancel, + SUM(IF(metricName = 'srs-cont-transfer-query', count, 0)) AS srs_cont_transfer_query, + SUM(IF(metricName = 'srs-cont-transfer-reject', count, 0)) AS srs_cont_transfer_reject, + SUM(IF(metricName = 'srs-cont-transfer-request', count, 0)) AS srs_cont_transfer_request, + SUM(IF(metricName = 'srs-cont-update', count, 0)) AS srs_cont_update, + -- Cross join a list of all TLDs against TLD-specific metrics and then + -- filter so that only metrics with that TLD or a NULL TLD are counted + -- towards a given TLD. + FROM ( + SELECT tldStr AS tld + FROM [latest_snapshot.Registry] + -- Include all real TLDs that are not in pre-delegation testing. + WHERE tldType = 'REAL' + OMIT RECORD IF SOME(tldStateTransitions.tldState = 'PDT') + ) AS Tld + CROSS JOIN ( + SELECT + tld, metricName, count + FROM + -- Dummy data source that ensures that all TLDs appear in report, + -- since they'll all have at least 1 joined row that survives. + (SELECT STRING(NULL) AS tld, STRING(NULL) AS metricName, 0 AS count), + -- BEGIN JOINED DATA SOURCES -- + +( + + -- Query for operational-registrars metric. + SELECT + allowedTlds AS tld, + 'operational-registrars' AS metricName, + INTEGER(COUNT(__key__.name)) AS count, + FROM [domain-registry:latest_snapshot.Registrar] + WHERE type = 'REAL' + AND creationTime < TIMESTAMP('2016-07-01') + GROUP BY tld + +), +( + + -- Query for all-ramped-up-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-ramped-up-registrars' AS metricName, + -- Sandbox OT&E registrar names can have either '-{1,2,3,4}' or '{,2,3}' + -- as suffixes - strip all of these off to get the "real" name. + INTEGER(EXACT_COUNT_DISTINCT( + REGEXP_EXTRACT(__key__.name, r'(.+?)(?:-?\d)?$'))) AS count, + FROM [domain-registry-sandbox:latest_snapshot.Registrar] + WHERE type = 'OTE' + AND creationTime < TIMESTAMP('2016-07-01') + +), +( + + -- Query for all-registrars metric. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + 'all-registrars' AS metricName, + INTEGER('None') AS count, + +), +( + + -- Query for WHOIS metrics. + SELECT + STRING(NULL) AS tld, -- Applies to all TLDs. + -- Whois queries over port 43 get forwarded by the proxy to /_dr/whois, + -- while web queries come in via /whois/. + CASE WHEN requestPath = '/_dr/whois' THEN 'whois-43-queries' + WHEN LEFT(requestPath, 7) = '/whois/' THEN 'web-whois-queries' + END AS metricName, + INTEGER(COUNT(requestPath)) AS count, + FROM ( + -- BEGIN LOGS QUERY -- + + -- Query AppEngine request logs for the report month. + SELECT + protoPayload.resource AS requestPath, + protoPayload.line.logMessage AS logMessage, + FROM + TABLE_DATE_RANGE_STRICT( + [appengine_logs.appengine_googleapis_com_request_log_], + TIMESTAMP('2016-06-01'), + -- End timestamp is inclusive, so subtract 1 second from the + -- timestamp representing the start of the next month. + DATE_ADD(TIMESTAMP('2016-07-01'), -1, 'SECOND')) + + -- END LOGS QUERY -- + ) + GROUP BY metricName + HAVING metricName IS NOT NULL + +), +( + + -- Query for DNS metrics. + SELECT + STRING(NULL) AS tld, + metricName, + -1 AS count, + FROM + (SELECT 'dns-udp-queries' AS metricName), + (SELECT 'dns-tcp-queries' AS metricName) + +), +( + + -- Query EPP XML messages and calculate SRS metrics. + SELECT + domainTld AS tld, + -- SRS metric names follow a set pattern corresponding to the EPP + -- protocol elements. First we extract the 'inner' command element in + -- EPP, e.g. , which is the resource type followed by + -- the standard EPP command type. To get the metric name, we add the + -- prefix 'srs-', abbreviate 'domain' as 'dom' and 'contact' as 'cont', + -- and replace ':' with '-' to produce 'srs-dom-create'. + -- + -- Transfers have subcommands indicated by an 'op' attribute, which we + -- extract and add as an extra suffix for transfer commands, so e.g. + -- 'srs-cont-transfer-approve'. Domain restores are domain updates + -- with a special element; if present, the command counts + -- under the srs-dom-rgp-restore-{request,report} metric (depending on + -- the value of the 'op' attribute) instead of srs-dom-update. + CONCAT( + 'srs-', + REPLACE(REPLACE(REPLACE( + CASE + WHEN NOT restoreOp IS NULL THEN CONCAT('domain-rgp-restore-', restoreOp) + WHEN commandType = 'transfer' THEN CONCAT(innerCommand, '-', commandOpArg) + ELSE innerCommand + END, + ':', '-'), 'domain', 'dom'), 'contact', 'cont') + ) AS metricName, + INTEGER(COUNT(xml)) AS count, + FROM ( + SELECT + -- Extract salient bits of the EPP XML using regexes. This is fairly + -- safe since the EPP gets schema-validated and pretty-printed before + -- getting logged, and so it looks something like this: + -- + -- + -- + -- , 'request' as the value of the + -- 'op' attribute of that element (if any), and 'domain:transfer' as + -- the inner command from the name of the subsequent element. + -- + -- Domain commands all have at least one element (more + -- than one for domain checks, but we just count the first), from which + -- we extract the domain TLD as everything after the first dot in the + -- element value. This won't work if the client mistakenly sends a + -- hostname (e.g. 'www.foo.example') as the domain name, but we prefer + -- this over taking everything after the last dot so that multipart + -- TLDs like 'co.uk' can be supported. + -- + -- Domain restores are indicated by an element, from + -- which we extract the value of the 'op' attribute. + -- + -- TODO(b/20725722): preprocess the XML in FlowRunner so we don't need + -- regex parsing of XML here (http://stackoverflow.com/a/1732454). + -- + REGEXP_EXTRACT(xml, '(?s).*?<([a-z]+)') AS commandType, + REGEXP_EXTRACT(xml, '(?s).*?<[a-z]+ op="(.+?)"') AS commandOpArg, + REGEXP_EXTRACT(xml, '(?s).*?<.+?>.*?<([a-z]+:[a-z]+)') AS innerCommand, + REGEXP_EXTRACT(xml, '[^.]+[.](.+)') AS domainTld, + REGEXP_EXTRACT(xml, '') AS restoreOp, + xml, + FROM ( + -- BEGIN EPP XML LOGS QUERY -- + + -- Query EPP request logs and extract the clientId and raw EPP XML. + SELECT + REGEXP_EXTRACT(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command\n\t.+\n\t(.+)\n') AS clientId, + REGEXP_EXTRACT(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command\n\t.+\n\t.+\n\t.+\n\t((?s).+)$') AS xml, + FROM ( + -- BEGIN LOGS QUERY -- + + -- Query AppEngine request logs for the report month. + SELECT + protoPayload.resource AS requestPath, + protoPayload.line.logMessage AS logMessage, + FROM + TABLE_DATE_RANGE_STRICT( + [appengine_logs.appengine_googleapis_com_request_log_], + TIMESTAMP('2016-06-01'), + -- End timestamp is inclusive, so subtract 1 second from the + -- timestamp representing the start of the next month. + DATE_ADD(TIMESTAMP('2016-07-01'), -1, 'SECOND')) + + -- END LOGS QUERY -- + ) + WHERE + -- EPP endpoints from the proxy, regtool, and console respectively. + requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr') + AND REGEXP_MATCH(logMessage, r'^(?:com.google.domain.registry.flows.FlowRunner run|com.google.domain.registry.util.FormattingLogger log|google.registry.util.FormattingLogger log): EPP Command') + + -- END EPP XML LOGS QUERY -- + ) + -- Filter to just XML that contains a element (no s). + WHERE xml CONTAINS '' + ) + -- Whitelist of EPP command types that we care about for metrics; + -- excludes login, logout, and poll. + WHERE commandType IN ('check', 'create', 'delete', 'info', 'renew', 'transfer', 'update') + GROUP BY tld, metricName + +) + -- END JOINED DATA SOURCES -- + ) AS TldMetrics + WHERE Tld.tld = TldMetrics.tld OR TldMetrics.tld IS NULL + GROUP BY tld + ORDER BY tld From d9596fa30c9887b2e1cc56f09a13d92072fd2e73 Mon Sep 17 00:00:00 2001 From: mcilwain Date: Fri, 15 Jul 2016 14:59:58 -0700 Subject: [PATCH 5/8] Add gTech command to get .app domain info ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127582475 --- javatests/google/registry/tools/BUILD | 4 +++- javatests/google/registry/tools/CommandTestCase.java | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/javatests/google/registry/tools/BUILD b/javatests/google/registry/tools/BUILD index 9583f12ff..f01c4d359 100644 --- a/javatests/google/registry/tools/BUILD +++ b/javatests/google/registry/tools/BUILD @@ -13,7 +13,9 @@ java_library( srcs = glob([ "*.java", ]), - resources = glob(["testdata/*.*"]), + resources = glob([ + "testdata/*.*", + ]), deps = [ "//java/com/google/common/annotations", "//java/com/google/common/base", diff --git a/javatests/google/registry/tools/CommandTestCase.java b/javatests/google/registry/tools/CommandTestCase.java index 5c26f82b2..6dac241dc 100644 --- a/javatests/google/registry/tools/CommandTestCase.java +++ b/javatests/google/registry/tools/CommandTestCase.java @@ -49,7 +49,8 @@ import org.mockito.runners.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public abstract class CommandTestCase { - private ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + private final ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + private final ByteArrayOutputStream stderr = new ByteArrayOutputStream(); protected C command; @@ -71,6 +72,7 @@ public abstract class CommandTestCase { RegistryToolEnvironment.UNITTEST.setup(); command = newCommandInstance(); System.setOut(new PrintStream(stdout)); + System.setErr(new PrintStream(stderr)); } void runCommandInEnvironment(RegistryToolEnvironment env, String... args) throws Exception { @@ -156,6 +158,13 @@ public abstract class CommandTestCase { } } + protected void assertInStderr(String... expected) throws Exception { + String stderror = new String(stderr.toByteArray(), UTF_8); + for (String line : expected) { + assertThat(stderror).contains(line); + } + } + void assertNotInStdout(String expected) throws Exception { assertThat(getStdoutAsString()).doesNotContain(expected); } From e72491e59b35b9e5f4f2022a149d5d53f1d91819 Mon Sep 17 00:00:00 2001 From: mountford Date: Fri, 15 Jul 2016 15:10:13 -0700 Subject: [PATCH 6/8] Replace to(Upper|Lower)Case with Ascii.to$1Case The presubmits are warning that toUpperCase() and toLowerCase() are locale-specific, and advise using Ascii.toUpperCase() and Ascii.toLowerCase() as a local-invariant alternative. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127583677 --- .../google/registry/config/RegistryEnvironment.java | 3 ++- .../google/registry/export/DatastoreBackupInfo.java | 3 ++- .../model/domain/fee/FeeCommandDescriptor.java | 3 ++- .../registry/model/domain/launch/LaunchNotice.java | 3 ++- .../registry/rde/DomainResourceToXjcConverter.java | 3 ++- java/google/registry/tmch/LordnLog.java | 13 ++++++++----- .../registry/tools/AllocateDomainCommand.java | 3 ++- java/google/registry/tools/CommandUtilities.java | 3 ++- java/google/registry/tools/ConvertIdnCommand.java | 3 ++- .../tools/DomainApplicationInfoCommand.java | 5 +++-- .../registry/tools/RegistryToolEnvironment.java | 3 ++- .../registry/tools/server/VerifyOteAction.java | 3 ++- java/google/registry/ui/forms/FormField.java | 12 +++++++----- .../registry/ui/server/RegistrarFormFields.java | 3 ++- java/google/registry/util/Idn.java | 4 ++-- java/google/registry/util/RegistrarUtils.java | 6 ++++-- java/google/registry/util/UrlFetchUtils.java | 5 +++-- .../dns/writer/dnsupdate/DnsUpdateWriterTest.java | 4 ++-- .../flows/domain/DomainTransferFlowTestCase.java | 5 +++-- .../google/registry/rde/RdeReportActionTest.java | 3 ++- .../google/registry/testing/DatastoreHelper.java | 3 ++- .../google/registry/testing/TaskQueueHelper.java | 7 ++++--- 22 files changed, 62 insertions(+), 38 deletions(-) diff --git a/java/google/registry/config/RegistryEnvironment.java b/java/google/registry/config/RegistryEnvironment.java index 0c194ca73..16aec6da6 100644 --- a/java/google/registry/config/RegistryEnvironment.java +++ b/java/google/registry/config/RegistryEnvironment.java @@ -15,6 +15,7 @@ package google.registry.config; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; import javax.annotation.Nullable; /** Registry environments. */ @@ -50,7 +51,7 @@ public enum RegistryEnvironment { /** Returns environment configured by system property {@value #PROPERTY}. */ public static RegistryEnvironment get() { - return valueOf(System.getProperty(PROPERTY, UNITTEST.name()).toUpperCase()); + return valueOf(Ascii.toUpperCase(System.getProperty(PROPERTY, UNITTEST.name()))); } /** diff --git a/java/google/registry/export/DatastoreBackupInfo.java b/java/google/registry/export/DatastoreBackupInfo.java index 312d4b220..a9a647827 100644 --- a/java/google/registry/export/DatastoreBackupInfo.java +++ b/java/google/registry/export/DatastoreBackupInfo.java @@ -21,6 +21,7 @@ import static org.joda.time.DateTimeZone.UTC; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.Text; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Ascii; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; @@ -140,7 +141,7 @@ public class DatastoreBackupInfo { "Status: " + getStatus(), "Started: " + startTime, "Ended: " + completeTime.orNull(), - "Duration: " + getRunningTime().toPeriod().toString().substring(2).toLowerCase(), + "Duration: " + Ascii.toLowerCase(getRunningTime().toPeriod().toString().substring(2)), "GCS: " + gcsFilename.orNull(), "Kinds: " + kinds, ""); diff --git a/java/google/registry/model/domain/fee/FeeCommandDescriptor.java b/java/google/registry/model/domain/fee/FeeCommandDescriptor.java index cc61f9944..d410cbf0c 100644 --- a/java/google/registry/model/domain/fee/FeeCommandDescriptor.java +++ b/java/google/registry/model/domain/fee/FeeCommandDescriptor.java @@ -14,6 +14,7 @@ package google.registry.model.domain.fee; +import com.google.common.base.Ascii; import com.google.common.base.CharMatcher; import google.registry.model.ImmutableObject; import javax.xml.bind.annotation.XmlAttribute; @@ -56,7 +57,7 @@ public class FeeCommandDescriptor extends ImmutableObject { // Require the xml string to be lowercase. if (command != null && CharMatcher.javaLowerCase().matchesAllOf(command)) { try { - return CommandName.valueOf(command.toUpperCase()); + return CommandName.valueOf(Ascii.toUpperCase(command)); } catch (IllegalArgumentException e) { // Swallow this and return UNKNOWN below because there's no matching CommandName. } diff --git a/java/google/registry/model/domain/launch/LaunchNotice.java b/java/google/registry/model/domain/launch/LaunchNotice.java index 92986d9cf..4858d14f9 100644 --- a/java/google/registry/model/domain/launch/LaunchNotice.java +++ b/java/google/registry/model/domain/launch/LaunchNotice.java @@ -20,6 +20,7 @@ import static com.google.common.io.BaseEncoding.base16; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import com.google.common.base.Ascii; import com.google.common.base.CharMatcher; import com.google.common.base.Optional; import com.google.common.primitives.Ints; @@ -101,7 +102,7 @@ public class LaunchNotice extends ImmutableObject { String tcnId = getNoticeId().getTcnId(); checkArgument(tcnId.length() == 27); - int checksum = Ints.fromByteArray(base16().decode(tcnId.substring(0, 8).toUpperCase())); + int checksum = Ints.fromByteArray(base16().decode(Ascii.toUpperCase(tcnId.substring(0, 8)))); String noticeId = tcnId.substring(8); checkArgument(CharMatcher.inRange('0', '9').matchesAllOf(noticeId)); diff --git a/java/google/registry/rde/DomainResourceToXjcConverter.java b/java/google/registry/rde/DomainResourceToXjcConverter.java index 921decf08..66f0bcc27 100644 --- a/java/google/registry/rde/DomainResourceToXjcConverter.java +++ b/java/google/registry/rde/DomainResourceToXjcConverter.java @@ -14,6 +14,7 @@ package google.registry.rde; +import com.google.common.base.Ascii; import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Ref; @@ -280,7 +281,7 @@ final class DomainResourceToXjcConverter { private static XjcDomainContactType convertDesignatedContact(DesignatedContact model) { XjcDomainContactType bean = new XjcDomainContactType(); ContactResource contact = model.getContactRef().get(); - bean.setType(XjcDomainContactAttrType.fromValue(model.getType().toString().toLowerCase())); + bean.setType(XjcDomainContactAttrType.fromValue(Ascii.toLowerCase(model.getType().toString()))); bean.setValue(contact.getContactId()); return bean; } diff --git a/java/google/registry/tmch/LordnLog.java b/java/google/registry/tmch/LordnLog.java index ca8f1c82a..3543c6a9d 100644 --- a/java/google/registry/tmch/LordnLog.java +++ b/java/google/registry/tmch/LordnLog.java @@ -18,6 +18,7 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.base.Ascii; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.google.re2j.Pattern; @@ -218,7 +219,7 @@ public final class LordnLog implements Iterable> // + , whether the LORDN file has been accepted for // processing by the TMDB. Possible values are "accepted" or // "rejected". - Status status = Status.valueOf(firstLine.get(4).toUpperCase()); + Status status = Status.valueOf(Ascii.toUpperCase(firstLine.get(4))); // + , whether the LORDN Log has any warning result // codes. Possible values are "no-warnings" or "warnings- @@ -229,8 +230,11 @@ public final class LordnLog implements Iterable> // processed in the LORDN file. int dnLines = Integer.parseInt(firstLine.get(6)); int actual = lines.size() - 2; - checkArgument(dnLines == actual, - "Line 1: Number of entries (%d) differs from declaration (%d)", actual, dnLines); + checkArgument( + dnLines == actual, + "Line 1: Number of entries (%s) differs from declaration (%s)", + String.valueOf(actual), + String.valueOf(dnLines)); // Second line contains headers: roid,result-code checkArgument(lines.get(1).equals("roid,result-code"), @@ -244,8 +248,7 @@ public final class LordnLog implements Iterable> "Line %d: Expected 2 elements, found %d", i + 1, currentLine.size())); String roid = currentLine.get(0); int code = Integer.parseInt(currentLine.get(1)); - Result result = checkNotNull(RESULTS.get(code), - "Line %d: Unknown result code: %d", i, code); + Result result = checkNotNull(RESULTS.get(code), "Line %s: Unknown result code: %s", i, code); builder.put(roid, result); } diff --git a/java/google/registry/tools/AllocateDomainCommand.java b/java/google/registry/tools/AllocateDomainCommand.java index 64ff8ea78..40dc3f34a 100644 --- a/java/google/registry/tools/AllocateDomainCommand.java +++ b/java/google/registry/tools/AllocateDomainCommand.java @@ -26,6 +26,7 @@ import static google.registry.tools.CommandUtilities.addHeader; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Splitter; @@ -134,7 +135,7 @@ final class AllocateDomainCommand extends MutatingEppToolCommand { ImmutableMap.Builder contactsMapBuilder = new ImmutableMap.Builder<>(); for (DesignatedContact contact : application.getContacts()) { contactsMapBuilder.put( - contact.getType().toString().toLowerCase(), + Ascii.toLowerCase(contact.getType().toString()), contact.getContactRef().get().getForeignKey()); } LaunchNotice launchNotice = application.getLaunchNotice(); diff --git a/java/google/registry/tools/CommandUtilities.java b/java/google/registry/tools/CommandUtilities.java index 95cd72689..553a3dbbd 100644 --- a/java/google/registry/tools/CommandUtilities.java +++ b/java/google/registry/tools/CommandUtilities.java @@ -14,6 +14,7 @@ package google.registry.tools; +import com.google.common.base.Ascii; import com.google.common.base.Strings; /** Container class for static utility methods. */ @@ -25,6 +26,6 @@ class CommandUtilities { /** Prompts for yes/no input using promptText, defaulting to no. */ static boolean promptForYes(String promptText) { - return System.console().readLine(promptText + " (y/N): ").toUpperCase().startsWith("Y"); + return Ascii.toUpperCase(System.console().readLine(promptText + " (y/N): ")).startsWith("Y"); } } diff --git a/java/google/registry/tools/ConvertIdnCommand.java b/java/google/registry/tools/ConvertIdnCommand.java index 10a45b910..c85411e57 100644 --- a/java/google/registry/tools/ConvertIdnCommand.java +++ b/java/google/registry/tools/ConvertIdnCommand.java @@ -19,6 +19,7 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.google.common.base.Ascii; import google.registry.tools.Command.GtechCommand; import google.registry.util.Idn; import java.io.IOException; @@ -37,7 +38,7 @@ final class ConvertIdnCommand implements Command, GtechCommand { public void run() throws IOException { for (String label : mainParameters) { if (label.startsWith(ACE_PREFIX)) { - System.out.println(Idn.toUnicode(label.toLowerCase())); + System.out.println(Idn.toUnicode(Ascii.toLowerCase(label))); } else { System.out.println(canonicalizeDomainName(label)); } diff --git a/java/google/registry/tools/DomainApplicationInfoCommand.java b/java/google/registry/tools/DomainApplicationInfoCommand.java index 70ecaee6a..891dcb804 100644 --- a/java/google/registry/tools/DomainApplicationInfoCommand.java +++ b/java/google/registry/tools/DomainApplicationInfoCommand.java @@ -18,6 +18,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import com.google.common.base.Ascii; import com.google.template.soy.data.SoyMapData; import google.registry.model.domain.launch.LaunchPhase; import google.registry.tools.Command.GtechCommand; @@ -53,8 +54,8 @@ final class DomainApplicationInfoCommand extends EppToolCommand implements Gtech @Override void initEppToolCommand() { - LaunchPhase launchPhase = - checkArgumentNotNull(LaunchPhase.fromValue(phase.toLowerCase()), "Illegal launch phase."); + LaunchPhase launchPhase = checkArgumentNotNull( + LaunchPhase.fromValue(Ascii.toLowerCase(phase)), "Illegal launch phase."); setSoyTemplate( DomainApplicationInfoSoyInfo.getInstance(), diff --git a/java/google/registry/tools/RegistryToolEnvironment.java b/java/google/registry/tools/RegistryToolEnvironment.java index 7567cbfcf..85a7ae2e7 100644 --- a/java/google/registry/tools/RegistryToolEnvironment.java +++ b/java/google/registry/tools/RegistryToolEnvironment.java @@ -17,6 +17,7 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import google.registry.config.RegistryEnvironment; @@ -61,7 +62,7 @@ enum RegistryToolEnvironment { * @see #get() */ static RegistryToolEnvironment parseFromArgs(String[] args) { - return valueOf(getFlagValue(args, FLAGS).toUpperCase()); + return valueOf(Ascii.toUpperCase(getFlagValue(args, FLAGS))); } /** diff --git a/java/google/registry/tools/server/VerifyOteAction.java b/java/google/registry/tools/server/VerifyOteAction.java index 5c8b63ad4..d6f990062 100644 --- a/java/google/registry/tools/server/VerifyOteAction.java +++ b/java/google/registry/tools/server/VerifyOteAction.java @@ -23,6 +23,7 @@ import static google.registry.util.CollectionUtils.isNullOrEmpty; import static google.registry.util.DomainNameUtils.ACE_PREFIX; import static java.util.Arrays.asList; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Predicate; @@ -250,7 +251,7 @@ public class VerifyOteAction implements Runnable, JsonAction { /** Returns a more human-readable translation of the enum constant. */ String description() { - return this.name().replace('_', ' ').toLowerCase(); + return Ascii.toLowerCase(this.name().replace('_', ' ')); } /** An {@link EppInput} might match multiple actions, so check if this action matches. */ diff --git a/java/google/registry/ui/forms/FormField.java b/java/google/registry/ui/forms/FormField.java index 0dc499b04..b8181a8f5 100644 --- a/java/google/registry/ui/forms/FormField.java +++ b/java/google/registry/ui/forms/FormField.java @@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import com.google.common.base.Ascii; import com.google.common.base.CharMatcher; import com.google.common.base.Function; import com.google.common.base.Functions; @@ -30,6 +31,7 @@ import com.google.common.collect.Range; import com.google.re2j.Pattern; import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import javax.annotation.Detainted; @@ -560,7 +562,7 @@ public final class FormField { @Nullable @Override public String apply(@Nullable String input) { - return input != null ? input.toUpperCase() : null; + return input != null ? input.toUpperCase(Locale.ENGLISH) : null; }}; private static final Function LOWERCASE_FUNCTION = @@ -568,7 +570,7 @@ public final class FormField { @Nullable @Override public String apply(@Nullable String input) { - return input != null ? input.toLowerCase() : null; + return input != null ? input.toLowerCase(Locale.ENGLISH) : null; }}; private static final Function REQUIRED_FUNCTION = @@ -587,8 +589,8 @@ public final class FormField { @Nullable @Override public Object apply(@Nullable Object input) { - return input instanceof CharSequence && ((CharSequence) input).length() == 0 - || input instanceof Collection && ((Collection) input).isEmpty() + return ((input instanceof CharSequence) && (((CharSequence) input).length() == 0)) + || ((input instanceof Collection) && ((Collection) input).isEmpty()) ? null : input; }}; @@ -709,7 +711,7 @@ public final class FormField { @Override public C apply(@Nullable O input) { try { - return input != null ? Enum.valueOf(enumClass, ((String) input).toUpperCase()) : null; + return input != null ? Enum.valueOf(enumClass, Ascii.toUpperCase((String) input)) : null; } catch (IllegalArgumentException e) { throw new FormFieldException( String.format("Enum %s does not contain '%s'", enumClass.getSimpleName(), input)); diff --git a/java/google/registry/ui/server/RegistrarFormFields.java b/java/google/registry/ui/server/RegistrarFormFields.java index b7f9f3761..8150ee387 100644 --- a/java/google/registry/ui/server/RegistrarFormFields.java +++ b/java/google/registry/ui/server/RegistrarFormFields.java @@ -19,6 +19,7 @@ import static com.google.common.collect.Range.atMost; import static com.google.common.collect.Range.closed; import static google.registry.util.DomainNameUtils.canonicalizeDomainName; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -354,7 +355,7 @@ public final class RegistrarFormFields { } for (String state : stateField.extractUntyped(args).asSet()) { if ("US".equals(countryCode)) { - state = state.toUpperCase(); + state = Ascii.toUpperCase(state); if (!StateCode.US_MAP.containsKey(state)) { throw new FormFieldException(stateField, "Unknown US state code."); } diff --git a/java/google/registry/util/Idn.java b/java/google/registry/util/Idn.java index ffb66dd20..dfc02f2ce 100644 --- a/java/google/registry/util/Idn.java +++ b/java/google/registry/util/Idn.java @@ -57,8 +57,8 @@ public final class Idn { } /** - * Translates a string from Unicode to ASCII Compatible Encoding (ACE), as defined by the ToASCII - * operation of RFC 3490. + * Translates a string from ASCII Compatible Encoding (ACE) to Unicode, as defined by the + * ToUnicode operation of RFC 3490. * *

This method always uses UTS46 transitional * processing. diff --git a/java/google/registry/util/RegistrarUtils.java b/java/google/registry/util/RegistrarUtils.java index f92901890..8f774bd40 100644 --- a/java/google/registry/util/RegistrarUtils.java +++ b/java/google/registry/util/RegistrarUtils.java @@ -16,11 +16,13 @@ package google.registry.util; import static com.google.common.base.CharMatcher.javaLetterOrDigit; +import com.google.common.base.Ascii; + /** Utilities for working with {@code Registrar} objects. */ public class RegistrarUtils { /** Strip out anything that isn't a letter or digit, and lowercase. */ public static String normalizeRegistrarName(String name) { - return javaLetterOrDigit().retainFrom(name).toLowerCase(); + return Ascii.toLowerCase(javaLetterOrDigit().retainFrom(name)); } /** @@ -29,6 +31,6 @@ public class RegistrarUtils { * in Datastore, and is suitable for use in email addresses. */ public static String normalizeClientId(String clientId) { - return clientId.toLowerCase().replaceAll("[^a-z0-9\\-]", ""); + return Ascii.toLowerCase(clientId).replaceAll("[^a-z0-9\\-]", ""); } } diff --git a/java/google/registry/util/UrlFetchUtils.java b/java/google/registry/util/UrlFetchUtils.java index 205d2fd27..30ff07108 100644 --- a/java/google/registry/util/UrlFetchUtils.java +++ b/java/google/registry/util/UrlFetchUtils.java @@ -26,6 +26,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.appengine.api.urlfetch.HTTPHeader; import com.google.appengine.api.urlfetch.HTTPRequest; import com.google.appengine.api.urlfetch.HTTPResponse; +import com.google.common.base.Ascii; import com.google.common.base.Optional; import com.google.common.net.MediaType; import java.security.NoSuchAlgorithmException; @@ -49,9 +50,9 @@ public final class UrlFetchUtils { } private static Optional getHeaderFirstInternal(Iterable hdrs, String name) { - name = name.toLowerCase(); + name = Ascii.toLowerCase(name); for (HTTPHeader header : hdrs) { - if (header.getName().toLowerCase().equals(name)) { + if (Ascii.toLowerCase(header.getName()).equals(name)) { return Optional.of(header.getValue()); } } diff --git a/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java b/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java index bc69bd281..686c04954 100644 --- a/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java +++ b/javatests/google/registry/dns/writer/dnsupdate/DnsUpdateWriterTest.java @@ -259,12 +259,12 @@ public class DnsUpdateWriterTest { Update update, String resourceName, int recordType, String... resourceData) { ArrayList expectedData = new ArrayList<>(); for (String resourceDatum : resourceData) { - expectedData.add(resourceDatum.toLowerCase()); + expectedData.add(resourceDatum); } ArrayList actualData = new ArrayList<>(); for (Record record : findUpdateRecords(update, resourceName, recordType)) { - actualData.add(record.rdataToString().toLowerCase()); + actualData.add(record.rdataToString()); } assertThat(actualData).containsExactlyElementsIn(expectedData); } diff --git a/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java b/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java index 3043a9988..9730ddbd4 100644 --- a/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java +++ b/javatests/google/registry/flows/domain/DomainTransferFlowTestCase.java @@ -26,6 +26,7 @@ import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.GenericEppResourceSubject.assertAboutEppResources; import static google.registry.util.DateTimeUtils.END_OF_TIME; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Ref; import google.registry.flows.Flow; @@ -113,7 +114,7 @@ public class DomainTransferFlowTestCase createTld(tld); contact = persistActiveContact("jd1234"); domain = new DomainResource.Builder() - .setRepoId("1-".concat(tld.toUpperCase())) + .setRepoId("1-".concat(Ascii.toUpperCase(tld))) .setFullyQualifiedDomainName(label + "." + tld) .setCurrentSponsorClientId("TheRegistrar") .setCreationClientId("TheRegistrar") @@ -157,7 +158,7 @@ public class DomainTransferFlowTestCase .build()); subordinateHost = persistResource( new HostResource.Builder() - .setRepoId("2-".concat(tld.toUpperCase())) + .setRepoId("2-".concat(Ascii.toUpperCase(tld))) .setFullyQualifiedHostName("ns1." + label + "." + tld) .setCurrentSponsorClientId("TheRegistrar") .setCreationClientId("TheRegistrar") diff --git a/javatests/google/registry/rde/RdeReportActionTest.java b/javatests/google/registry/rde/RdeReportActionTest.java index 142f3768f..ac7227d61 100644 --- a/javatests/google/registry/rde/RdeReportActionTest.java +++ b/javatests/google/registry/rde/RdeReportActionTest.java @@ -38,6 +38,7 @@ import com.google.appengine.api.urlfetch.URLFetchService; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.GcsServiceFactory; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import google.registry.config.RegistryConfig; @@ -194,7 +195,7 @@ public class RdeReportActionTest { private static ImmutableMap mapifyHeaders(Iterable headers) { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); for (HTTPHeader header : headers) { - builder.put(header.getName().replace('-', '_').toUpperCase(), header.getValue()); + builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue()); } return builder.build(); } diff --git a/javatests/google/registry/testing/DatastoreHelper.java b/javatests/google/registry/testing/DatastoreHelper.java index a4ef02327..6b046c782 100644 --- a/javatests/google/registry/testing/DatastoreHelper.java +++ b/javatests/google/registry/testing/DatastoreHelper.java @@ -36,6 +36,7 @@ import static google.registry.util.ResourceUtils.readResourceUtf8; import static java.util.Arrays.asList; import static org.joda.money.CurrencyUnit.USD; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Predicate; @@ -360,7 +361,7 @@ public class DatastoreHelper { } public static void createTld(String tld, ImmutableSortedMap tldStates) { - createTld(tld, tld.replaceFirst(ACE_PREFIX_REGEX, "").toUpperCase(), tldStates); + createTld(tld, Ascii.toUpperCase(tld.replaceFirst(ACE_PREFIX_REGEX, "")), tldStates); } public static void createTld( diff --git a/javatests/google/registry/testing/TaskQueueHelper.java b/javatests/google/registry/testing/TaskQueueHelper.java index d2c786ae5..020e2987b 100644 --- a/javatests/google/registry/testing/TaskQueueHelper.java +++ b/javatests/google/registry/testing/TaskQueueHelper.java @@ -30,6 +30,7 @@ import static java.util.Arrays.asList; import com.google.appengine.api.taskqueue.dev.QueueStateInfo; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.HeaderWrapper; import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo; +import com.google.common.base.Ascii; import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Predicate; @@ -94,7 +95,7 @@ public class TaskQueueHelper { public TaskMatcher header(String name, String value) { // Lowercase for case-insensitive comparison. - expected.headers.put(name.toLowerCase(), value); + expected.headers.put(Ascii.toLowerCase(name), value); return this; } @@ -310,7 +311,7 @@ public class TaskQueueHelper { for (HeaderWrapper header : info.getHeaders()) { // Lowercase header name for comparison since HTTP // header names are case-insensitive. - headerBuilder.put(header.getKey().toLowerCase(), header.getValue()); + headerBuilder.put(Ascii.toLowerCase(header.getKey()), header.getValue()); } this.headers = headerBuilder.build(); ImmutableMultimap.Builder inputParams = new ImmutableMultimap.Builder<>(); @@ -319,7 +320,7 @@ public class TaskQueueHelper { inputParams.putAll(UriParameters.parse(query)); } if (headers.containsEntry( - HttpHeaders.CONTENT_TYPE.toLowerCase(), MediaType.FORM_DATA.toString())) { + Ascii.toLowerCase(HttpHeaders.CONTENT_TYPE), MediaType.FORM_DATA.toString())) { inputParams.putAll(UriParameters.parse(info.getBody())); } this.params = inputParams.build(); From 41817985ff0d5cb173b0f446b4cd4abe2798aa54 Mon Sep 17 00:00:00 2001 From: mountford Date: Mon, 18 Jul 2016 08:22:02 -0700 Subject: [PATCH 7/8] RDAP: Implement new status codes The RDAP community is adding new values to the list of valid status codes, so that all EPP status codes can map to corresponding RDAP ones. The RDAP code has been updated accordingly. For more information, see https://tools.ietf.org/html/draft-ietf-regext-epp-rdap-status-mapping-01. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127718882 --- .../registry/rdap/RdapJsonFormatter.java | 53 ++++++++++++++----- .../registry/rdap/testdata/rdap_domain.json | 8 +-- .../rdap/testdata/rdap_domain_unicode.json | 8 +-- .../rdap/testdata/rdap_multiple_domains.json | 16 +++--- .../rdap/testdata/rdapjson_domain_full.json | 8 +-- .../rdapjson_domain_no_nameservers.json | 8 +-- 6 files changed, 64 insertions(+), 37 deletions(-) diff --git a/java/google/registry/rdap/RdapJsonFormatter.java b/java/google/registry/rdap/RdapJsonFormatter.java index d439b8d02..263fb1c07 100644 --- a/java/google/registry/rdap/RdapJsonFormatter.java +++ b/java/google/registry/rdap/RdapJsonFormatter.java @@ -85,8 +85,9 @@ public class RdapJsonFormatter { static final String NOTICES = "notices"; private static final String REMARKS = "remarks"; - /** Status values specified in RFC 7483 § 10.2.2. */ private enum RdapStatus { + + // Status values specified in RFC 7483 § 10.2.2. VALIDATED("validated"), RENEW_PROHIBITED("renew prohibited"), UPDATE_PROHIBITED("update prohibited"), @@ -104,7 +105,26 @@ public class RdapJsonFormatter { PENDING_RENEW("pending renew"), PENDING_TRANSFER("pending transfer"), PENDING_UPDATE("pending update"), - PENDING_DELETE("pending delete"); + PENDING_DELETE("pending delete"), + + // Additional status values defined in + // https://tools.ietf.org/html/draft-ietf-regext-epp-rdap-status-mapping-01. + ADD_PERIOD("add period"), + AUTO_RENEW_PERIOD("auto renew period"), + CLIENT_DELETE_PROHIBITED("client delete prohibited"), + CLIENT_HOLD("client hold"), + CLIENT_RENEW_PROHIBITED("client renew prohibited"), + CLIENT_TRANSFER_PROHIBITED("client transfer prohibited"), + CLIENT_UPDATE_PROHIBITED("client update prohibited"), + PENDING_RESTORE("pending restore"), + REDEMPTION_PERIOD("redemption period"), + RENEW_PERIOD("renew period"), + SERVER_DELETE_PROHIBITED("server deleted prohibited"), + SERVER_RENEW_PROHIBITED("server renew prohibited"), + SERVER_TRANSFER_PROHIBITED("server transfer prohibited"), + SERVER_UPDATE_PROHIBITED("server update prohibited"), + SERVER_HOLD("server hold"), + TRANSFER_PERIOD("transfer period"); /** Value as it appears in RDAP messages. */ private final String rfc7483String; @@ -123,23 +143,30 @@ public class RdapJsonFormatter { private static final ImmutableMap statusToRdapStatusMap = Maps.immutableEnumMap( new ImmutableMap.Builder() - .put(StatusValue.CLIENT_DELETE_PROHIBITED, RdapStatus.DELETE_PROHIBITED) - .put(StatusValue.CLIENT_HOLD, RdapStatus.INACTIVE) - .put(StatusValue.CLIENT_RENEW_PROHIBITED, RdapStatus.RENEW_PROHIBITED) - .put(StatusValue.CLIENT_TRANSFER_PROHIBITED, RdapStatus.TRANSFER_PROHIBITED) - .put(StatusValue.CLIENT_UPDATE_PROHIBITED, RdapStatus.UPDATE_PROHIBITED) + // StatusValue.ADD_PERIOD not defined in our system + // StatusValue.AUTO_RENEW_PERIOD not defined in our system + .put(StatusValue.CLIENT_DELETE_PROHIBITED, RdapStatus.CLIENT_DELETE_PROHIBITED) + .put(StatusValue.CLIENT_HOLD, RdapStatus.CLIENT_HOLD) + .put(StatusValue.CLIENT_RENEW_PROHIBITED, RdapStatus.CLIENT_RENEW_PROHIBITED) + .put(StatusValue.CLIENT_TRANSFER_PROHIBITED, RdapStatus.CLIENT_TRANSFER_PROHIBITED) + .put(StatusValue.CLIENT_UPDATE_PROHIBITED, RdapStatus.CLIENT_UPDATE_PROHIBITED) .put(StatusValue.INACTIVE, RdapStatus.INACTIVE) .put(StatusValue.LINKED, RdapStatus.ASSOCIATED) .put(StatusValue.OK, RdapStatus.ACTIVE) .put(StatusValue.PENDING_CREATE, RdapStatus.PENDING_CREATE) .put(StatusValue.PENDING_DELETE, RdapStatus.PENDING_DELETE) + // StatusValue.PENDING_RENEW not defined in our system + // StatusValue.PENDING_RESTORE not defined in our system .put(StatusValue.PENDING_TRANSFER, RdapStatus.PENDING_TRANSFER) .put(StatusValue.PENDING_UPDATE, RdapStatus.PENDING_UPDATE) - .put(StatusValue.SERVER_DELETE_PROHIBITED, RdapStatus.DELETE_PROHIBITED) - .put(StatusValue.SERVER_HOLD, RdapStatus.INACTIVE) - .put(StatusValue.SERVER_RENEW_PROHIBITED, RdapStatus.RENEW_PROHIBITED) - .put(StatusValue.SERVER_TRANSFER_PROHIBITED, RdapStatus.TRANSFER_PROHIBITED) - .put(StatusValue.SERVER_UPDATE_PROHIBITED, RdapStatus.UPDATE_PROHIBITED) + // StatusValue.REDEMPTION_PERIOD not defined in our system + // StatusValue.RENEW_PERIOD not defined in our system + .put(StatusValue.SERVER_DELETE_PROHIBITED, RdapStatus.SERVER_DELETE_PROHIBITED) + .put(StatusValue.SERVER_HOLD, RdapStatus.SERVER_HOLD) + .put(StatusValue.SERVER_RENEW_PROHIBITED, RdapStatus.SERVER_RENEW_PROHIBITED) + .put(StatusValue.SERVER_TRANSFER_PROHIBITED, RdapStatus.SERVER_TRANSFER_PROHIBITED) + .put(StatusValue.SERVER_UPDATE_PROHIBITED, RdapStatus.SERVER_UPDATE_PROHIBITED) + // StatusValue.TRANSFER_PERIOD not defined in our system .build()); /** Role values specified in RFC 7483 § 10.2.4. */ @@ -189,7 +216,7 @@ public class RdapJsonFormatter { } } - /** Map of EPP status values to the RDAP equivalents. */ + /** Map of EPP event values to the RDAP equivalents. */ private static final ImmutableMap historyEntryTypeToRdapEventActionMap = Maps.immutableEnumMap( diff --git a/javatests/google/registry/rdap/testdata/rdap_domain.json b/javatests/google/registry/rdap/testdata/rdap_domain.json index 8ab26c32c..c8441b57e 100644 --- a/javatests/google/registry/rdap/testdata/rdap_domain.json +++ b/javatests/google/registry/rdap/testdata/rdap_domain.json @@ -1,9 +1,9 @@ { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "handle": "%HANDLE%", "links": [ diff --git a/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json b/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json index d97e09edb..e6787e609 100644 --- a/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json +++ b/javatests/google/registry/rdap/testdata/rdap_domain_unicode.json @@ -1,9 +1,9 @@ { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "unicodeName": "%NAME%", "handle": "%HANDLE%", diff --git a/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json b/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json index a8f24a0f9..35b480286 100644 --- a/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json +++ b/javatests/google/registry/rdap/testdata/rdap_multiple_domains.json @@ -2,10 +2,10 @@ "domainSearchResults": [ { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "handle": "21-EXAMPLE", "links": [ @@ -361,10 +361,10 @@ }, { "status": [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "handle": "C-LOL", "links": [ diff --git a/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json b/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json index 85fb8e076..6befaae8e 100644 --- a/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json +++ b/javatests/google/registry/rdap/testdata/rdapjson_domain_full.json @@ -5,10 +5,10 @@ "unicodeName" : "cat.みんな", "status" : [ - "delete prohibited", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", + "server update prohibited" ], "links" : [ diff --git a/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json b/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json index 49469ca72..fe91a0ee7 100644 --- a/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json +++ b/javatests/google/registry/rdap/testdata/rdapjson_domain_no_nameservers.json @@ -5,11 +5,11 @@ "unicodeName" : "fish.みんな", "status" : [ - "delete prohibited", + "client delete prohibited", + "client renew prohibited", + "client transfer prohibited", "inactive", - "renew prohibited", - "transfer prohibited", - "update prohibited" + "server update prohibited" ], "links" : [ From ac6b147bf510a6bcbc7d8efe3379dca4870211c4 Mon Sep 17 00:00:00 2001 From: Wolfgang Meyers Date: Mon, 18 Jul 2016 15:17:38 -0700 Subject: [PATCH 8/8] Add ContactResource import utility logic ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=127765049 --- java/google/registry/rde/RdeImportUtils.java | 101 ++++++++++++ .../registry/rde/RdeImportUtilsTest.java | 153 ++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 java/google/registry/rde/RdeImportUtils.java create mode 100644 javatests/google/registry/rde/RdeImportUtilsTest.java diff --git a/java/google/registry/rde/RdeImportUtils.java b/java/google/registry/rde/RdeImportUtils.java new file mode 100644 index 000000000..f3ce5dc23 --- /dev/null +++ b/java/google/registry/rde/RdeImportUtils.java @@ -0,0 +1,101 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.rde; + +import static com.google.common.base.Preconditions.checkState; + +import com.googlecode.objectify.Key; +import com.googlecode.objectify.Work; +import google.registry.model.contact.ContactResource; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.index.ForeignKeyIndex; +import google.registry.model.ofy.Ofy; +import google.registry.util.Clock; +import google.registry.util.FormattingLogger; +import javax.inject.Inject; + +/** Utility functions for escrow file import. */ +public final class RdeImportUtils { + + private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); + + private final Ofy ofy; + private final Clock clock; + + @Inject + public RdeImportUtils(Ofy ofy, Clock clock) { + this.ofy = ofy; + this.clock = clock; + } + + /** + * Imports a contact from an escrow file. + * + *

The contact will only be imported if it has not been previously imported. + * + *

If the contact is imported, {@link ForeignKeyIndex} and {@link EppResourceIndex} are also + * created. + * + * @return true if the contact was created or updated, false otherwise. + */ + public boolean importContact(final ContactResource resource) { + return ofy.transact( + new Work() { + @Override + public Boolean run() { + ContactResource existing = ofy.load().key(Key.create(resource)).now(); + if (existing == null) { + ForeignKeyIndex existingForeignKeyIndex = + ForeignKeyIndex.load( + ContactResource.class, resource.getContactId(), clock.nowUtc()); + // foreign key index should not exist, since existing contact was not found. + checkState( + existingForeignKeyIndex == null, + String.format( + "New contact resource has existing foreign key index. " + + "contactId=%s, repoId=%s", + resource.getContactId(), resource.getRepoId())); + ofy.save().entity(resource); + ofy.save().entity(ForeignKeyIndex.create(resource, resource.getDeletionTime())); + ofy.save().entity(EppResourceIndex.create(Key.create(resource))); + logger.infofmt( + "Imported contact resource - ROID=%s, id=%s", + resource.getRepoId(), resource.getContactId()); + return true; + } else if (!existing.getRepoId().equals(resource.getRepoId())) { + logger.warningfmt( + "Existing contact with same contact id but different ROID. " + + "contactId=%s, existing ROID=%s, new ROID=%s", + resource.getContactId(), existing.getRepoId(), resource.getRepoId()); + } + return false; + } + }); + } +} diff --git a/javatests/google/registry/rde/RdeImportUtilsTest.java b/javatests/google/registry/rde/RdeImportUtilsTest.java new file mode 100644 index 000000000..29bbc2c06 --- /dev/null +++ b/javatests/google/registry/rde/RdeImportUtilsTest.java @@ -0,0 +1,153 @@ +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2016 The Domain Registry Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.rde; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.testing.DatastoreHelper.persistResource; + +import com.google.common.base.Predicate; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableList; +import com.googlecode.objectify.Key; +import com.googlecode.objectify.Work; +import google.registry.model.EppResource; +import google.registry.model.contact.ContactResource; +import google.registry.model.index.EppResourceIndex; +import google.registry.model.index.EppResourceIndexBucket; +import google.registry.model.index.ForeignKeyIndex; +import google.registry.testing.AppEngineRule; +import google.registry.testing.FakeClock; +import google.registry.testing.ShardableTestCase; +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link RdeImportUtils} */ +@RunWith(JUnit4.class) +public class RdeImportUtilsTest extends ShardableTestCase { + + @Rule + public final AppEngineRule appEngine = AppEngineRule.builder() + .withDatastore() + .build(); + + private RdeImportUtils rdeImportUtils; + private FakeClock clock; + + @Before + public void before() { + clock = new FakeClock(); + clock.setTo(DateTime.now()); + rdeImportUtils = new RdeImportUtils(ofy(), clock); + } + + /** Verifies import of a contact that has not been previously imported */ + @Test + public void testImportNewContact() { + ContactResource newContact = buildNewContact(); + assertThat(rdeImportUtils.importContact(newContact)).isTrue(); + assertEppResourceIndexEntityFor(newContact); + assertForeignKeyIndexFor(newContact); + + // verify the new contact was saved + ContactResource saved = getContact("TEST-123"); + assertThat(saved).isNotNull(); + assertThat(saved.getContactId()).isEqualTo(newContact.getContactId()); + assertThat(saved.getEmailAddress()).isEqualTo(newContact.getEmailAddress()); + assertThat(saved.getLastEppUpdateTime()).isEqualTo(newContact.getLastEppUpdateTime()); + } + + /** Verifies that a contact will not be imported more than once */ + @Test + public void testImportExistingContact() { + ContactResource newContact = buildNewContact(); + persistResource(newContact); + ContactResource updatedContact = newContact.asBuilder() + .setLastEppUpdateTime(newContact.getLastEppUpdateTime().plusSeconds(1)) + .build(); + assertThat(rdeImportUtils.importContact(updatedContact)).isFalse(); + + // verify the updated contact was saved + ContactResource saved = getContact("TEST-123"); + assertThat(saved).isNotNull(); + assertThat(saved.getContactId()).isEqualTo(newContact.getContactId()); + assertThat(saved.getEmailAddress()).isEqualTo(newContact.getEmailAddress()); + assertThat(saved.getLastEppUpdateTime()).isEqualTo(newContact.getLastEppUpdateTime()); + } + + private static ContactResource buildNewContact() { + return new ContactResource.Builder() + .setContactId("sh8013") + .setEmailAddress("jdoe@example.com") + .setLastEppUpdateTime(DateTime.parse("2010-10-10T00:00:00.000Z")) + .setRepoId("TEST-123") + .build(); + } + + private static ContactResource getContact(String repoId) { + final Key key = Key.create(ContactResource.class, repoId); + return ofy().transact(new Work() { + + @Override + public ContactResource run() { + return ofy().load().key(key).now(); + }}); + } + + /** + * Confirms that a ForeignKeyIndex exists in the datastore for a given resource. + */ + private static void assertForeignKeyIndexFor(final T resource) { + assertThat(ForeignKeyIndex.load(resource.getClass(), resource.getForeignKey(), DateTime.now())) + .isNotNull(); + } + + /** + * Confirms that an EppResourceIndex entity exists in datastore for a given resource. + */ + private static void assertEppResourceIndexEntityFor(final T resource) { + ImmutableList indices = FluentIterable + .from(ofy().load() + .type(EppResourceIndex.class) + .filter("kind", Key.getKind(resource.getClass()))) + .filter(new Predicate() { + @Override + public boolean apply(EppResourceIndex index) { + return index.getReference().get().equals(resource); + }}) + .toList(); + assertThat(indices).hasSize(1); + assertThat(indices.get(0).getBucket()) + .isEqualTo(EppResourceIndexBucket.getBucketKey(Key.create(resource))); + } +}