mirror of
https://github.com/google/nomulus.git
synced 2025-05-14 00:17:20 +02:00
Delete obsolete billing code
Now that we've verified the new Beam billing pipeline works, we can delete the old manual commands we used to use. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=184707182
This commit is contained in:
larryruili
jianglai
parent
2e62ad2658
commit
5f218b4a8b
10 changed files with 0 additions and 701 deletions
|
|
@ -1,53 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.tools;
|
|
||||||
|
|
||||||
import com.google.common.util.concurrent.ListenableFuture;
|
|
||||||
import google.registry.bigquery.BigqueryConnection.DestinationTable;
|
|
||||||
import java.util.concurrent.ExecutionException;
|
|
||||||
|
|
||||||
/** Container class for static utility methods for Bigquery commands. */
|
|
||||||
final class BigqueryCommandUtilities {
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Handler that takes a DestinationTable future and waits on its completion, printing generic
|
|
||||||
* success/failure messages and wrapping any exception thrown in a TableCreationException.
|
|
||||||
*/
|
|
||||||
static void handleTableCreation(
|
|
||||||
String tableDescription,
|
|
||||||
ListenableFuture<DestinationTable> tableFuture) throws TableCreationException {
|
|
||||||
System.err.printf("Creating %s...\n", tableDescription);
|
|
||||||
try {
|
|
||||||
DestinationTable table = tableFuture.get();
|
|
||||||
System.err.printf(" - Success: created %s.\n", table.getStringReference());
|
|
||||||
} catch (Exception e) {
|
|
||||||
Throwable error = e;
|
|
||||||
if (e instanceof ExecutionException) {
|
|
||||||
error = e.getCause();
|
|
||||||
}
|
|
||||||
String errorMessage =
|
|
||||||
String.format("Failed to create %s: %s", tableDescription, error.getMessage());
|
|
||||||
System.err.printf(" - %s\n", errorMessage);
|
|
||||||
throw new TableCreationException(errorMessage, error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Exception thrown if any error occurs during a table creation stage. */
|
|
||||||
static class TableCreationException extends Exception {
|
|
||||||
TableCreationException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,164 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.tools;
|
|
||||||
|
|
||||||
import static com.google.common.base.Preconditions.checkArgument;
|
|
||||||
import static google.registry.tools.BigqueryCommandUtilities.handleTableCreation;
|
|
||||||
import static google.registry.util.ResourceUtils.readResourceUtf8;
|
|
||||||
|
|
||||||
import com.beust.jcommander.Parameter;
|
|
||||||
import com.beust.jcommander.Parameters;
|
|
||||||
import com.google.common.base.Joiner;
|
|
||||||
import google.registry.bigquery.BigqueryUtils.TableType;
|
|
||||||
import google.registry.tools.BigqueryCommandUtilities.TableCreationException;
|
|
||||||
import google.registry.util.SqlTemplate;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Optional;
|
|
||||||
|
|
||||||
/** Command to make synthetic billing tables and views in Bigquery. */
|
|
||||||
@Parameters(separators = " =", commandDescription = "Make synthetic billing tables in Bigquery")
|
|
||||||
final class MakeBillingTablesCommand extends BigqueryCommand {
|
|
||||||
|
|
||||||
@Parameter(
|
|
||||||
names = "--source_dataset",
|
|
||||||
description = "Name of the dataset containing the source entity tables.")
|
|
||||||
private String sourceDatasetId;
|
|
||||||
|
|
||||||
// TODO(b/14297938): we should be picking up the TLDs to bill automatically based on the tldType
|
|
||||||
// and tldState rather than having to manually pass them in.
|
|
||||||
@Parameter(
|
|
||||||
names = "--tlds",
|
|
||||||
description = "TLD(s) to include in billing data table",
|
|
||||||
required = true)
|
|
||||||
private List<String> tlds;
|
|
||||||
|
|
||||||
private static final SqlTemplate CURRENCY_TABLE_SQL = getSql("currency_table.sql");
|
|
||||||
private static final SqlTemplate REGISTRAR_DATA_SQL = getSql("registrar_data_view.sql");
|
|
||||||
private static final SqlTemplate REGISTRAR_ACCOUNT_DATA_SQL =
|
|
||||||
getSql("registrar_account_data_view.sql");
|
|
||||||
private static final SqlTemplate REGISTRY_DATA_SQL = getSql("registry_data_view.sql");
|
|
||||||
private static final SqlTemplate BILLING_DATA_SQL = getSql("billing_data_view.sql");
|
|
||||||
|
|
||||||
/** Runs the main billing table/view creation logic. */
|
|
||||||
@Override
|
|
||||||
public void runWithBigquery() throws Exception {
|
|
||||||
// Make the source dataset default to the default destination dataset if it has not been set.
|
|
||||||
sourceDatasetId = Optional.ofNullable(sourceDatasetId).orElse(bigquery().getDatasetId());
|
|
||||||
checkArgument(!tlds.isEmpty(), "Must specify at least 1 TLD to include in billing data table");
|
|
||||||
// TODO(b/19016191): Should check that input tables exist up front, and avoid later errors.
|
|
||||||
try {
|
|
||||||
makeCurrencyTable();
|
|
||||||
makeRegistrarView();
|
|
||||||
makeRegistrarAccountView();
|
|
||||||
makeRegistryView();
|
|
||||||
makeBillingView();
|
|
||||||
} catch (TableCreationException e) {
|
|
||||||
// Swallow since we already will have printed an error message.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Generates a table of currency information. */
|
|
||||||
// TODO(b/19016720): once there is a registry-wide currency, this method should compute the set of
|
|
||||||
// currencies needed for the active registries, look up the exponent for each currency from Joda
|
|
||||||
// CurrencyUnit data, and then auto-generate the query to construct this table.
|
|
||||||
// NB: "exponent" is the ISO 4217 name for the number of decimal places used by a currency.
|
|
||||||
private void makeCurrencyTable() throws Exception {
|
|
||||||
handleTableCreation(
|
|
||||||
"currency table",
|
|
||||||
bigquery().query(
|
|
||||||
CURRENCY_TABLE_SQL
|
|
||||||
.build(),
|
|
||||||
bigquery().buildDestinationTable("Currency")
|
|
||||||
.description("Generated table of currency information.")
|
|
||||||
.build()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates a view of registrar data, used as an intermediate so that the invoice generation
|
|
||||||
* stage doesn't have to refer all the way back to the original managed backup dataset.
|
|
||||||
*/
|
|
||||||
private void makeRegistrarView() throws Exception {
|
|
||||||
handleTableCreation(
|
|
||||||
"registrar data view",
|
|
||||||
bigquery().query(
|
|
||||||
REGISTRAR_DATA_SQL
|
|
||||||
.put("SOURCE_DATASET", sourceDatasetId)
|
|
||||||
.build(),
|
|
||||||
bigquery().buildDestinationTable("RegistrarData")
|
|
||||||
.description("Synthetic view of registrar information.")
|
|
||||||
.type(TableType.VIEW)
|
|
||||||
.build()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates a view of registrar account data, which includes the billing account id and the
|
|
||||||
* currency used.
|
|
||||||
*
|
|
||||||
* <p>The generated view is similar to the one generated by {@link #makeRegistrarView()}, but it
|
|
||||||
* uses currency-specific billing account id and does not include allowed TLDs.
|
|
||||||
*/
|
|
||||||
private void makeRegistrarAccountView() throws Exception {
|
|
||||||
handleTableCreation(
|
|
||||||
"registrar account data view",
|
|
||||||
bigquery()
|
|
||||||
.query(
|
|
||||||
REGISTRAR_ACCOUNT_DATA_SQL.put("SOURCE_DATASET", sourceDatasetId).build(),
|
|
||||||
bigquery()
|
|
||||||
.buildDestinationTable("RegistrarAccountData")
|
|
||||||
.description(
|
|
||||||
"Synthetic view of registrar information "
|
|
||||||
+ "with currency-specific billing account id.")
|
|
||||||
.type(TableType.VIEW)
|
|
||||||
.build()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Generates a view of registry data to feed into later views. */
|
|
||||||
private void makeRegistryView() throws Exception {
|
|
||||||
handleTableCreation(
|
|
||||||
"registry data view",
|
|
||||||
bigquery().query(
|
|
||||||
REGISTRY_DATA_SQL
|
|
||||||
.put("SOURCE_DATASET", sourceDatasetId)
|
|
||||||
.build(),
|
|
||||||
bigquery().buildDestinationTable("RegistryData")
|
|
||||||
.description("Synthetic view of registry information.")
|
|
||||||
.type(TableType.VIEW)
|
|
||||||
.build()));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates a view of consolidated billing information that includes currency conversions,
|
|
||||||
* registrar details, and cancellation flags on top of the original BillingEvent.OneTime data.
|
|
||||||
*/
|
|
||||||
private void makeBillingView() throws Exception {
|
|
||||||
handleTableCreation(
|
|
||||||
"billing data view",
|
|
||||||
bigquery().query(
|
|
||||||
BILLING_DATA_SQL
|
|
||||||
.put("SOURCE_DATASET", sourceDatasetId)
|
|
||||||
.put("DEST_DATASET", bigquery().getDatasetId())
|
|
||||||
.put("TLDS", Joiner.on(",").join(tlds))
|
|
||||||
.build(),
|
|
||||||
bigquery().buildDestinationTable("BillingData")
|
|
||||||
.description("Synthetic view of consolidated billing information.")
|
|
||||||
.type(TableType.VIEW)
|
|
||||||
.build()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static SqlTemplate getSql(String filename) {
|
|
||||||
return SqlTemplate.create(
|
|
||||||
readResourceUtf8(MakeBillingTablesCommand.class, "sql/" + filename));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,102 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.tools;
|
|
||||||
|
|
||||||
import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
|
|
||||||
|
|
||||||
import com.beust.jcommander.Parameter;
|
|
||||||
import com.beust.jcommander.Parameters;
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
|
||||||
import google.registry.export.PublishDetailReportAction;
|
|
||||||
import google.registry.model.registrar.Registrar;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Command to publish a given billing detail report to a registrar's Drive folder.
|
|
||||||
*/
|
|
||||||
@Parameters(separators = " =", commandDescription = "Publish detail report for a registrar")
|
|
||||||
public class PublishDetailReportCommand extends ConfirmingCommand
|
|
||||||
implements ServerSideCommand {
|
|
||||||
|
|
||||||
@Parameter(
|
|
||||||
names = "--registrar_id",
|
|
||||||
description = "Client identifier of the registrar to publish the report for",
|
|
||||||
required = true)
|
|
||||||
private String clientId;
|
|
||||||
|
|
||||||
@Parameter(
|
|
||||||
names = "--report_name",
|
|
||||||
description = "Name of the detail report (without directory prefixes)",
|
|
||||||
required = true)
|
|
||||||
private String reportName;
|
|
||||||
|
|
||||||
@Parameter(
|
|
||||||
names = "--gcs_bucket",
|
|
||||||
description = "Name of the GCS bucket that holds billing output files.")
|
|
||||||
private String gcsBucket = "domain-registry-billing";
|
|
||||||
|
|
||||||
@Parameter(
|
|
||||||
names = "--gcs_folder",
|
|
||||||
description = "GCS folder under which report was exported.",
|
|
||||||
required = true)
|
|
||||||
private String gcsFolder;
|
|
||||||
|
|
||||||
private static final String DRIVE_FOLDER_URL_TEMPLATE =
|
|
||||||
"https://drive.google.com/corp/drive/#folders/%s";
|
|
||||||
|
|
||||||
private String gcsFolderPrefix;
|
|
||||||
private String driveFolderUrl;
|
|
||||||
|
|
||||||
private Connection connection;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void setConnection(Connection connection) {
|
|
||||||
this.connection = connection;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void init() throws Exception {
|
|
||||||
// Append a trailing slash to the GCS folder (if there is one, and if it doesn't already end
|
|
||||||
// in a slash) to get the "folder prefix" version.
|
|
||||||
// TODO(b/18611424): Fix PublishDetailReportAction to take fewer error-prone parameters.
|
|
||||||
gcsFolderPrefix =
|
|
||||||
(gcsFolder.isEmpty() || gcsFolder.endsWith("/")) ? gcsFolder : gcsFolder + "/";
|
|
||||||
Registrar registrar =
|
|
||||||
checkArgumentPresent(
|
|
||||||
Registrar.loadByClientId(clientId), "Registrar with ID %s not found", clientId);
|
|
||||||
driveFolderUrl = String.format(DRIVE_FOLDER_URL_TEMPLATE, registrar.getDriveFolderId());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected String prompt() {
|
|
||||||
String gcsFile = String.format("gs://%s/%s%s", gcsBucket, gcsFolderPrefix, reportName);
|
|
||||||
return "Publish detail report:\n"
|
|
||||||
+ " - Registrar: " + clientId + "\n"
|
|
||||||
+ " - Drive folder: " + driveFolderUrl + "\n"
|
|
||||||
+ " - GCS file: " + gcsFile;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected String execute() throws Exception {
|
|
||||||
final ImmutableMap<String, String> params = ImmutableMap.of(
|
|
||||||
PublishDetailReportAction.REGISTRAR_ID_PARAM, clientId,
|
|
||||||
PublishDetailReportAction.DETAIL_REPORT_NAME_PARAM, reportName,
|
|
||||||
PublishDetailReportAction.GCS_FOLDER_PREFIX_PARAM, gcsFolderPrefix,
|
|
||||||
PublishDetailReportAction.GCS_BUCKET_PARAM, gcsBucket);
|
|
||||||
Map<String, Object> response =
|
|
||||||
connection.sendJson(PublishDetailReportAction.PATH, params);
|
|
||||||
return "Success! Published report with drive file ID: " + response.get("driveId");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -99,10 +99,8 @@ public final class RegistryTool {
|
||||||
.put("lock_domain", LockDomainCommand.class)
|
.put("lock_domain", LockDomainCommand.class)
|
||||||
.put("login", LoginCommand.class)
|
.put("login", LoginCommand.class)
|
||||||
.put("logout", LogoutCommand.class)
|
.put("logout", LogoutCommand.class)
|
||||||
.put("make_billing_tables", MakeBillingTablesCommand.class)
|
|
||||||
.put("pending_escrow", PendingEscrowCommand.class)
|
.put("pending_escrow", PendingEscrowCommand.class)
|
||||||
.put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
|
.put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
|
||||||
.put("publish_detail_report", PublishDetailReportCommand.class)
|
|
||||||
.put("registrar_activity_report", RegistrarActivityReportCommand.class)
|
.put("registrar_activity_report", RegistrarActivityReportCommand.class)
|
||||||
.put("registrar_contact", RegistrarContactCommand.class)
|
.put("registrar_contact", RegistrarContactCommand.class)
|
||||||
.put("remove_ip_address", RemoveIpAddressCommand.class)
|
.put("remove_ip_address", RemoveIpAddressCommand.class)
|
||||||
|
|
|
||||||
|
|
@ -1,156 +0,0 @@
|
||||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
--
|
|
||||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
-- you may not use this file except in compliance with the License.
|
|
||||||
-- You may obtain a copy of the License at
|
|
||||||
--
|
|
||||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
--
|
|
||||||
-- Unless required by applicable law or agreed to in writing, software
|
|
||||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
-- See the License for the specific language governing permissions and
|
|
||||||
-- limitations under the License.
|
|
||||||
|
|
||||||
-- Billing Data View SQL
|
|
||||||
--
|
|
||||||
-- This query post-processes the OneTime billing events and then annotates
|
|
||||||
-- the resulting data with additional information from the Registrar,
|
|
||||||
-- DomainBase, Currency, and Cancellation tables.
|
|
||||||
SELECT
|
|
||||||
id,
|
|
||||||
BillingEvent.billingTime AS billingTime,
|
|
||||||
BillingEvent.eventTime AS eventTime,
|
|
||||||
BillingEvent.clientId AS registrarId,
|
|
||||||
Registrar.billingIdentifier AS billingId,
|
|
||||||
RegistrarAccount.billingAccountId AS billingAccountId,
|
|
||||||
BillingEvent.tld AS tld,
|
|
||||||
IF(
|
|
||||||
CONCAT(',', BillingEvent.flags, ',') CONTAINS (',ALLOCATION,'),
|
|
||||||
'ALLOCATE',
|
|
||||||
BillingEvent.reason) AS action,
|
|
||||||
BillingEvent.targetId AS domain,
|
|
||||||
BillingEvent.domainRepoId AS repositoryId,
|
|
||||||
periodYears AS years,
|
|
||||||
BillingEvent.currency AS currency,
|
|
||||||
amountMinor,
|
|
||||||
REGEXP_EXTRACT(cost, ' (.+)') AS amountString,
|
|
||||||
ROUND(amountMinor * Currency.conversionToUsd, 2) AS estimatedUsd,
|
|
||||||
flags,
|
|
||||||
Cancellation.cancellationId IS NOT NULL AS cancelled,
|
|
||||||
Cancellation.cancellationTime AS cancellationTime,
|
|
||||||
FROM (
|
|
||||||
SELECT
|
|
||||||
__key__.id AS id,
|
|
||||||
billingTime,
|
|
||||||
eventTime,
|
|
||||||
clientId,
|
|
||||||
tld,
|
|
||||||
reason,
|
|
||||||
targetId,
|
|
||||||
REGEXP_EXTRACT(__key__.path, '"DomainBase", "([^"]+)"') AS domainRepoId,
|
|
||||||
periodYears,
|
|
||||||
cost,
|
|
||||||
-- TODO(b/19031545): Find cleaner way to parse out currency and amount.
|
|
||||||
-- Parse out the currency code as the substring of 'cost' up to the space.
|
|
||||||
REGEXP_EXTRACT(cost, '(.+) ') AS currency,
|
|
||||||
-- Parse out the amount of minor units by stripping out non-digit chars
|
|
||||||
-- (i.e. currency, space, and period) and then converting to integer.
|
|
||||||
INTEGER(REGEXP_REPLACE(cost, r'\D+', '')) AS amountMinor,
|
|
||||||
-- Convert repeated flags field into flat comma-delimited string field,
|
|
||||||
-- excluding the internal-only flag 'SYNTHETIC'.
|
|
||||||
GROUP_CONCAT(IF(flags != 'SYNTHETIC', flags, NULL)) WITHIN RECORD AS flags,
|
|
||||||
-- Cancellations for recurring events will point to the recurring event's
|
|
||||||
-- key, which is stored in cancellationMatchingBillingEvent. The path
|
|
||||||
-- contains kind, id, and domainRepoId, all of which must match, so just
|
|
||||||
-- use the path.
|
|
||||||
COALESCE(cancellationMatchingBillingEvent.path, __key__.path)
|
|
||||||
AS cancellationMatchingPath,
|
|
||||||
FROM (
|
|
||||||
SELECT
|
|
||||||
*,
|
|
||||||
-- Count everything after first dot as TLD (to support multi-part TLDs).
|
|
||||||
REGEXP_EXTRACT(targetId, r'[.](.+)') AS tld,
|
|
||||||
FROM [%SOURCE_DATASET%.OneTime])
|
|
||||||
WHERE
|
|
||||||
-- Filter out prober data.
|
|
||||||
tld IN
|
|
||||||
(SELECT tld FROM [%DEST_DATASET%.RegistryData] WHERE type = 'REAL')
|
|
||||||
) AS BillingEvent
|
|
||||||
|
|
||||||
-- Join to pick up billing ID from registrar table.
|
|
||||||
LEFT JOIN EACH (
|
|
||||||
SELECT
|
|
||||||
__key__.name AS clientId,
|
|
||||||
billingIdentifier,
|
|
||||||
FROM
|
|
||||||
[%SOURCE_DATASET%.Registrar]
|
|
||||||
) AS Registrar
|
|
||||||
ON
|
|
||||||
BillingEvent.clientId = Registrar.clientId
|
|
||||||
|
|
||||||
-- Join to pick up currency specific registrar account id from registrar
|
|
||||||
-- account view.
|
|
||||||
LEFT JOIN EACH (
|
|
||||||
SELECT
|
|
||||||
registrarId,
|
|
||||||
billingAccountId,
|
|
||||||
currency
|
|
||||||
FROM
|
|
||||||
[%DEST_DATASET%.RegistrarAccountData]) AS RegistrarAccount
|
|
||||||
ON
|
|
||||||
BillingEvent.clientId = RegistrarAccount.registrarId
|
|
||||||
AND BillingEvent.currency = RegistrarAccount.currency
|
|
||||||
|
|
||||||
-- Join to pick up cancellations for billing events.
|
|
||||||
LEFT JOIN EACH (
|
|
||||||
SELECT
|
|
||||||
__key__.id AS cancellationId,
|
|
||||||
-- Coalesce matching fields from refOneTime and refRecurring (only one or
|
|
||||||
-- the other will ever be populated) for joining against referenced event.
|
|
||||||
COALESCE(refOneTime.path, refRecurring.path) AS cancelledEventPath,
|
|
||||||
eventTime AS cancellationTime,
|
|
||||||
billingTime AS cancellationBillingTime,
|
|
||||||
FROM (
|
|
||||||
SELECT
|
|
||||||
*,
|
|
||||||
-- Count everything after first dot as TLD (to support multi-part TLDs).
|
|
||||||
REGEXP_EXTRACT(targetId, r'[.](.+)') AS tld,
|
|
||||||
FROM
|
|
||||||
[%SOURCE_DATASET%.Cancellation])
|
|
||||||
WHERE
|
|
||||||
-- Filter out prober data.
|
|
||||||
tld IN
|
|
||||||
(SELECT tld FROM [%DEST_DATASET%.RegistryData] WHERE type = 'REAL')
|
|
||||||
) AS Cancellation
|
|
||||||
ON
|
|
||||||
BillingEvent.cancellationMatchingPath = Cancellation.cancelledEventPath
|
|
||||||
-- Require billing times to match so that cancellations for Recurring events
|
|
||||||
-- only apply to the specific recurrence being cancelled.
|
|
||||||
AND BillingEvent.billingTime = Cancellation.cancellationBillingTime
|
|
||||||
|
|
||||||
-- Join to pick up currency conversion factor.
|
|
||||||
LEFT JOIN EACH (
|
|
||||||
SELECT
|
|
||||||
currency,
|
|
||||||
conversionToUsd,
|
|
||||||
FROM
|
|
||||||
[%DEST_DATASET%.Currency]
|
|
||||||
) AS Currency
|
|
||||||
ON
|
|
||||||
BillingEvent.currency = Currency.currency
|
|
||||||
|
|
||||||
WHERE
|
|
||||||
-- Filter down to whitelisted TLDs that are "billable".
|
|
||||||
-- TODO(b/18092292): determine this automatically.
|
|
||||||
BillingEvent.tld IN
|
|
||||||
(SELECT tld FROM FLATTEN((
|
|
||||||
-- %TLDS% is passed in as a comma-delimited string of TLDs.
|
|
||||||
SELECT SPLIT('%TLDS%') AS tld FROM (SELECT 1 as unused)), tld))
|
|
||||||
|
|
||||||
-- Sort rows to show the latest billed items first.
|
|
||||||
ORDER BY
|
|
||||||
billingTime DESC,
|
|
||||||
-- Break ties in billing time using ID then TLD, to be deterministic.
|
|
||||||
id,
|
|
||||||
tld
|
|
||||||
|
|
@ -1,22 +0,0 @@
|
||||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
--
|
|
||||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
-- you may not use this file except in compliance with the License.
|
|
||||||
-- You may obtain a copy of the License at
|
|
||||||
--
|
|
||||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
--
|
|
||||||
-- Unless required by applicable law or agreed to in writing, software
|
|
||||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
-- See the License for the specific language governing permissions and
|
|
||||||
-- limitations under the License.
|
|
||||||
|
|
||||||
-- Currency Table Creation SQL
|
|
||||||
--
|
|
||||||
-- This query generates a static table of currency information.
|
|
||||||
SELECT
|
|
||||||
currency, conversionToUsd, exponent
|
|
||||||
FROM
|
|
||||||
(SELECT 'JPY' AS currency, 0.0098 AS conversionToUsd, 0 AS exponent),
|
|
||||||
(SELECT 'USD' AS currency, 0.0100 AS conversionToUsd, 2 AS exponent)
|
|
||||||
|
|
@ -1,50 +0,0 @@
|
||||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
--
|
|
||||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
-- you may not use this file except in compliance with the License.
|
|
||||||
-- You may obtain a copy of the License at
|
|
||||||
--
|
|
||||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
--
|
|
||||||
-- Unless required by applicable law or agreed to in writing, software
|
|
||||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
-- See the License for the specific language governing permissions and
|
|
||||||
-- limitations under the License.
|
|
||||||
|
|
||||||
-- Registrar Account Data View SQL
|
|
||||||
--
|
|
||||||
-- This query lists registrar IDs with billing account IDs and corresponding
|
|
||||||
-- currencies.
|
|
||||||
--
|
|
||||||
-- The table that contains both billing account IDs and currencies as repeated
|
|
||||||
-- fields are first flattened to two separate tables, with IDs and currencies
|
|
||||||
-- in each table, and the corresponding row numbers, partitioned over registrar.
|
|
||||||
-- The row numbers are used to join the two tables together, restoring the
|
|
||||||
-- original mapping between IDs and currencies.
|
|
||||||
SELECT
|
|
||||||
I.registrarId AS registrarId,
|
|
||||||
-- Apply no-op STRING() function to keep BigQuery schema transformation logic
|
|
||||||
-- from wanting different names in direct query vs save-as-view.
|
|
||||||
STRING(C.billingAccountMap.currency) AS currency,
|
|
||||||
STRING(I.billingAccountMap.accountId) AS billingAccountId,
|
|
||||||
FROM (
|
|
||||||
SELECT
|
|
||||||
__key__.name AS registrarId,
|
|
||||||
billingAccountMap.accountId,
|
|
||||||
ROW_NUMBER() OVER (PARTITION BY registrarId) AS pos
|
|
||||||
FROM
|
|
||||||
FLATTEN([%SOURCE_DATASET%.Registrar], billingAccountMap.accountId)) AS I
|
|
||||||
JOIN (
|
|
||||||
SELECT
|
|
||||||
__key__.name AS registrarId,
|
|
||||||
billingAccountMap.currency,
|
|
||||||
ROW_NUMBER() OVER (PARTITION BY registrarId) AS pos
|
|
||||||
FROM
|
|
||||||
FLATTEN([%SOURCE_DATASET%.Registrar], billingAccountMap.currency)) AS C
|
|
||||||
ON
|
|
||||||
I.registrarId == C.registrarId
|
|
||||||
AND I.pos == C.pos
|
|
||||||
ORDER BY
|
|
||||||
registrarId,
|
|
||||||
I.pos
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
--
|
|
||||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
-- you may not use this file except in compliance with the License.
|
|
||||||
-- You may obtain a copy of the License at
|
|
||||||
--
|
|
||||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
--
|
|
||||||
-- Unless required by applicable law or agreed to in writing, software
|
|
||||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
-- See the License for the specific language governing permissions and
|
|
||||||
-- limitations under the License.
|
|
||||||
|
|
||||||
-- Registrar Data View SQL
|
|
||||||
--
|
|
||||||
-- This query lists each registrar ID with type, billing ID, and allowed TLDs.
|
|
||||||
--
|
|
||||||
-- Note that there is one row per registrar (allowedTlds is repeated), versus
|
|
||||||
-- registrar_account_data_view.sql which has multiple rows per registrar.
|
|
||||||
SELECT
|
|
||||||
__key__.name AS registrarId,
|
|
||||||
type,
|
|
||||||
billingIdentifier AS billingId,
|
|
||||||
allowedTlds
|
|
||||||
FROM
|
|
||||||
[%SOURCE_DATASET%.Registrar]
|
|
||||||
-- Note: We can't ORDER BY registrarId here because ORDER/GROUP BY will
|
|
||||||
-- flatten results, and the allowedTlds field above is repeated.
|
|
||||||
-- TODO(b/19031339): Add "ORDER BY" if the BigQuery known issue is fixed.
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
-- Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
--
|
|
||||||
-- Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
-- you may not use this file except in compliance with the License.
|
|
||||||
-- You may obtain a copy of the License at
|
|
||||||
--
|
|
||||||
-- http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
--
|
|
||||||
-- Unless required by applicable law or agreed to in writing, software
|
|
||||||
-- distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
-- See the License for the specific language governing permissions and
|
|
||||||
-- limitations under the License.
|
|
||||||
|
|
||||||
-- Registry Data View SQL
|
|
||||||
--
|
|
||||||
-- This query lists registry fields necessary for billing.
|
|
||||||
--
|
|
||||||
-- TODO(b/20764952): extend this view to support timed transition properties.
|
|
||||||
-- TODO(b/18092292): add a column for whether the TLD is "billable" or not.
|
|
||||||
SELECT
|
|
||||||
tldStr AS tld,
|
|
||||||
tldType AS type,
|
|
||||||
FROM
|
|
||||||
[%SOURCE_DATASET%.Registry]
|
|
||||||
ORDER BY
|
|
||||||
tld
|
|
||||||
|
|
@ -1,95 +0,0 @@
|
||||||
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package google.registry.tools;
|
|
||||||
|
|
||||||
import static org.mockito.Matchers.anyMapOf;
|
|
||||||
import static org.mockito.Matchers.anyString;
|
|
||||||
import static org.mockito.Matchers.eq;
|
|
||||||
import static org.mockito.Mockito.verify;
|
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
|
||||||
import google.registry.tools.ServerSideCommand.Connection;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.mockito.Mock;
|
|
||||||
|
|
||||||
/** Unit tests for {@link PublishDetailReportCommand}. */
|
|
||||||
public class PublishDetailReportCommandTest extends CommandTestCase<PublishDetailReportCommand> {
|
|
||||||
|
|
||||||
@Mock
|
|
||||||
private Connection connection;
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void init() throws Exception {
|
|
||||||
command.setConnection(connection);
|
|
||||||
when(connection.sendJson(anyString(), anyMapOf(String.class, Object.class)))
|
|
||||||
.thenReturn(ImmutableMap.of("driveId", "some123id"));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_normalUsage() throws Exception {
|
|
||||||
runCommandForced(
|
|
||||||
"--registrar_id=TheRegistrar",
|
|
||||||
"--report_name=report.csv",
|
|
||||||
"--gcs_bucket=mah-buckit",
|
|
||||||
"--gcs_folder=some/folder");
|
|
||||||
verify(connection).sendJson(
|
|
||||||
eq("/_dr/publishDetailReport"),
|
|
||||||
eq(ImmutableMap.of(
|
|
||||||
"registrar", "TheRegistrar",
|
|
||||||
"report", "report.csv",
|
|
||||||
"gcsFolder", "some/folder/",
|
|
||||||
"bucket", "mah-buckit")));
|
|
||||||
assertInStdout("Success!");
|
|
||||||
assertInStdout("some123id");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_gcsFolderWithTrailingSlash() throws Exception {
|
|
||||||
runCommandForced(
|
|
||||||
"--registrar_id=TheRegistrar",
|
|
||||||
"--report_name=report.csv",
|
|
||||||
"--gcs_bucket=mah-buckit",
|
|
||||||
"--gcs_folder=some/folder/");
|
|
||||||
verify(connection).sendJson(
|
|
||||||
eq("/_dr/publishDetailReport"),
|
|
||||||
eq(ImmutableMap.of(
|
|
||||||
"registrar", "TheRegistrar",
|
|
||||||
"report", "report.csv",
|
|
||||||
"gcsFolder", "some/folder/",
|
|
||||||
"bucket", "mah-buckit")));
|
|
||||||
assertInStdout("Success!");
|
|
||||||
assertInStdout("some123id");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testSuccess_emptyGcsFolder() throws Exception {
|
|
||||||
runCommandForced(
|
|
||||||
"--registrar_id=TheRegistrar",
|
|
||||||
"--report_name=report.csv",
|
|
||||||
"--gcs_bucket=mah-buckit",
|
|
||||||
"--gcs_folder=");
|
|
||||||
verify(connection).sendJson(
|
|
||||||
eq("/_dr/publishDetailReport"),
|
|
||||||
eq(ImmutableMap.of(
|
|
||||||
"registrar", "TheRegistrar",
|
|
||||||
"report", "report.csv",
|
|
||||||
"gcsFolder", "",
|
|
||||||
"bucket", "mah-buckit")));
|
|
||||||
assertInStdout("Success!");
|
|
||||||
assertInStdout("some123id");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue