// Copyright 2017 The Nomulus Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package google.registry.beam; import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import google.registry.reporting.billing.BillingModule; import google.registry.util.FormattingLogger; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.stream.Collectors; import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.coders.AtomicCoder; import org.apache.beam.sdk.coders.Coder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; /** * A POJO representing a single billable event, parsed from a {@code SchemaAndRecord}. * *

This is a trivially serializable class that allows Beam to transform the results of a Bigquery * query into a standard Java representation, giving us the type guarantees and ease of manipulation * Bigquery lacks, while localizing any Bigquery-side failures to the {@link #parseFromRecord} * function. */ @AutoValue public abstract class BillingEvent implements Serializable { private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass(); private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss zzz"); private static final ImmutableList FIELD_NAMES = ImmutableList.of( "id", "billingTime", "eventTime", "registrarId", "billingId", "tld", "action", "domain", "repositoryId", "years", "currency", "amount", "flags"); /** Returns the unique Objectify ID for the {@code OneTime} associated with this event. */ abstract long id(); /** Returns the UTC DateTime this event becomes billable. */ abstract ZonedDateTime billingTime(); /** Returns the UTC DateTime this event was generated. */ abstract ZonedDateTime eventTime(); /** Returns the billed registrar's name. */ abstract String registrarId(); /** Returns the billed registrar's billing account key. */ abstract String billingId(); /** Returns the tld this event was generated for. */ abstract String tld(); /** Returns the billable action this event was generated for (i.e. RENEW, CREATE, TRANSFER...) */ abstract String action(); /** Returns the fully qualified domain name this event was generated for. */ abstract String domain(); /** Returns the unique RepoID associated with the billed domain. */ abstract String repositoryId(); /** Returns the number of years this billing event is made out for. */ abstract int years(); /** Returns the 3-letter currency code for the billing event (i.e. USD or JPY.) */ abstract String currency(); /** Returns the cost associated with this billing event. */ abstract double amount(); /** Returns a list of space-delimited flags associated with the event. */ abstract String flags(); /** * Constructs a {@code BillingEvent} from a {@code SchemaAndRecord}. * * @see * Apache AVRO GenericRecord */ static BillingEvent parseFromRecord(SchemaAndRecord schemaAndRecord) { checkFieldsNotNull(schemaAndRecord); GenericRecord record = schemaAndRecord.getRecord(); return create( // We need to chain parsers off extractField because GenericRecord only returns // Objects, which contain a string representation of their underlying types. Long.parseLong(extractField(record, "id")), // Bigquery provides UNIX timestamps with microsecond precision. Instant.ofEpochMilli(Long.parseLong(extractField(record, "billingTime")) / 1000) .atZone(ZoneId.of("UTC")), Instant.ofEpochMilli(Long.parseLong(extractField(record, "eventTime")) / 1000) .atZone(ZoneId.of("UTC")), extractField(record, "registrarId"), extractField(record, "billingId"), extractField(record, "tld"), extractField(record, "action"), extractField(record, "domain"), extractField(record, "repositoryId"), Integer.parseInt(extractField(record, "years")), extractField(record, "currency"), Double.parseDouble(extractField(record, "amount")), extractField(record, "flags")); } /** * Creates a concrete {@code BillingEvent}. * *

This should only be used outside this class for testing- instances of {@code BillingEvent} * should otherwise come from {@link #parseFromRecord}. */ @VisibleForTesting static BillingEvent create( long id, ZonedDateTime billingTime, ZonedDateTime eventTime, String registrarId, String billingId, String tld, String action, String domain, String repositoryId, int years, String currency, double amount, String flags) { return new AutoValue_BillingEvent( id, billingTime, eventTime, registrarId, billingId, tld, action, domain, repositoryId, years, currency, amount, flags); } static String getHeader() { return FIELD_NAMES.stream().collect(Collectors.joining(",")); } /** * Generates the filename associated with this {@code BillingEvent}. * *

When modifying this function, take care to ensure that there's no way to generate an illegal * filepath with the arguments, such as "../sensitive_info". */ String toFilename(String yearMonth) { return String.format( "%s_%s_%s_%s", BillingModule.DETAIL_REPORT_PREFIX, yearMonth, registrarId(), tld()); } /** Generates a CSV representation of this {@code BillingEvent}. */ String toCsv() { return Joiner.on(",") .join( ImmutableList.of( id(), DATE_TIME_FORMATTER.format(billingTime()), DATE_TIME_FORMATTER.format(eventTime()), registrarId(), billingId(), tld(), action(), domain(), repositoryId(), years(), currency(), String.format("%.2f", amount()), // Strip out the 'synthetic' flag, which is internal only. flags().replace("SYNTHETIC", "").trim())); } /** Returns the grouping key for this {@code BillingEvent}, to generate the overall invoice. */ InvoiceGroupingKey getInvoiceGroupingKey() { return new AutoValue_BillingEvent_InvoiceGroupingKey( billingTime().toLocalDate().withDayOfMonth(1).toString(), billingTime().toLocalDate().withDayOfMonth(1).plusYears(years()).minusDays(1).toString(), billingId(), String.format("%s - %s", registrarId(), tld()), String.format("%s | TLD: %s | TERM: %d-year", action(), tld(), years()), amount(), currency(), ""); } /** Key for each {@code BillingEvent}, when aggregating for the overall invoice. */ @AutoValue abstract static class InvoiceGroupingKey implements Serializable { private static final ImmutableList INVOICE_HEADERS = ImmutableList.of( "StartDate", "EndDate", "ProductAccountKey", "Amount", "AmountCurrency", "BillingProductCode", "SalesChannel", "LineItemType", "UsageGroupingKey", "Quantity", "Description", "UnitPrice", "UnitPriceCurrency", "PONumber"); /** Returns the first day this invoice is valid, in yyyy-MM-dd format. */ abstract String startDate(); /** Returns the last day this invoice is valid, in yyyy-MM-dd format. */ abstract String endDate(); /** Returns the billing account id, which is the {@code BillingEvent.billingId}. */ abstract String productAccountKey(); /** Returns the invoice grouping key, which is in the format "registrarId - tld". */ abstract String usageGroupingKey(); /** Returns a description of the item, formatted as "action | TLD: tld | TERM: n-year." */ abstract String description(); /** Returns the cost per invoice item. */ abstract Double unitPrice(); /** Returns the 3-digit currency code the unit price uses. */ abstract String unitPriceCurrency(); /** Returns the purchase order number for the item, blank for most registrars. */ abstract String poNumber(); /** Generates the CSV header for the overall invoice. */ static String invoiceHeader() { return Joiner.on(",").join(INVOICE_HEADERS); } /** Generates a CSV representation of n aggregate billing events. */ String toCsv(Long quantity) { double totalPrice = unitPrice() * quantity; return Joiner.on(",") .join( ImmutableList.of( startDate(), endDate(), productAccountKey(), String.format("%.2f", totalPrice), unitPriceCurrency(), "10125", "1", "PURCHASE", usageGroupingKey(), String.format("%d", quantity), description(), String.format("%.2f", unitPrice()), unitPriceCurrency(), poNumber())); } /** Coder that provides deterministic (de)serialization for {@code InvoiceGroupingKey}. */ static class InvoiceGroupingKeyCoder extends AtomicCoder { @Override public void encode(InvoiceGroupingKey value, OutputStream outStream) throws IOException { Coder stringCoder = StringUtf8Coder.of(); stringCoder.encode(value.startDate(), outStream); stringCoder.encode(value.endDate(), outStream); stringCoder.encode(value.productAccountKey(), outStream); stringCoder.encode(value.usageGroupingKey(), outStream); stringCoder.encode(value.description(), outStream); stringCoder.encode(String.valueOf(value.unitPrice()), outStream); stringCoder.encode(value.unitPriceCurrency(), outStream); stringCoder.encode(value.poNumber(), outStream); } @Override public InvoiceGroupingKey decode(InputStream inStream) throws IOException { Coder stringCoder = StringUtf8Coder.of(); return new AutoValue_BillingEvent_InvoiceGroupingKey( stringCoder.decode(inStream), stringCoder.decode(inStream), stringCoder.decode(inStream), stringCoder.decode(inStream), stringCoder.decode(inStream), Double.parseDouble(stringCoder.decode(inStream)), stringCoder.decode(inStream), stringCoder.decode(inStream)); } } } /** Extracts a string representation of a field in a {@code GenericRecord}. */ private static String extractField(GenericRecord record, String fieldName) { return String.valueOf(record.get(fieldName)); } /** * Checks that no expected fields in the record are missing. * *

Note that this simply makes sure the field is not null; it may still generate a parse error * in {@code parseFromRecord}. */ private static void checkFieldsNotNull(SchemaAndRecord schemaAndRecord) { GenericRecord record = schemaAndRecord.getRecord(); ImmutableList nullFields = FIELD_NAMES .stream() .filter(fieldName -> record.get(fieldName) == null) .collect(ImmutableList.toImmutableList()); if (!nullFields.isEmpty()) { logger.severefmt( "Found unexpected null value(s) in field(s) %s for record %s", Joiner.on(", ").join(nullFields), record.toString()); throw new IllegalStateException("Read null value from Bigquery query"); } } }