Add publish functionality to billing pipeline

This closes the end-to-end billing pipeline, allowing us to share generated detail reports with registrars via Drive and e-mail the invoicing team a link to the generated invoice.

This also factors out the email configs from ICANN reporting into the common 'misc' config, since we'll likely need alert e-mails for future periodic tasks.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=180805972
This commit is contained in:
larryruili 2018-01-04 09:16:51 -08:00 committed by jianglai
parent 27f12b9390
commit ab5e16ab67
25 changed files with 721 additions and 95 deletions

View file

@ -9,6 +9,7 @@ java_library(
srcs = glob(["*.java"]),
resources = glob(["sql/*"]),
deps = [
"//java/google/registry/billing",
"//java/google/registry/config",
"//java/google/registry/util",
"@com_google_apis_google_api_services_bigquery",

View file

@ -18,6 +18,7 @@ import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import google.registry.billing.BillingModule;
import google.registry.util.FormattingLogger;
import java.io.IOException;
import java.io.InputStream;
@ -173,7 +174,8 @@ public abstract class BillingEvent implements Serializable {
* filepath with the arguments, such as "../sensitive_info".
*/
String toFilename(String yearMonth) {
return String.format("invoice_details_%s_%s_%s", yearMonth, registrarId(), tld());
return String.format(
"%s_%s_%s_%s", BillingModule.DETAIL_REPORT_PREFIX, yearMonth, registrarId(), tld());
}
/** Generates a CSV representation of this {@code BillingEvent}. */

View file

@ -16,6 +16,7 @@ package google.registry.beam;
import google.registry.beam.BillingEvent.InvoiceGroupingKey;
import google.registry.beam.BillingEvent.InvoiceGroupingKey.InvoiceGroupingKeyCoder;
import google.registry.billing.BillingModule;
import google.registry.config.RegistryConfig.Config;
import java.io.Serializable;
import javax.inject.Inject;
@ -130,7 +131,11 @@ public class InvoicingPipeline implements Serializable {
.to(
NestedValueProvider.of(
yearMonthProvider,
yearMonth -> String.format("%s/results/CRR-INV-%s", beamBucket, yearMonth)))
// TODO(larryruili): Replace with billing bucket after verifying 2017-12 output.
yearMonth ->
String.format(
"%s/results/%s-%s",
beamBucket, BillingModule.OVERALL_INVOICE_PREFIX, yearMonth)))
.withHeader(InvoiceGroupingKey.invoiceHeader())
.withoutSharding()
.withSuffix(".csv");
@ -140,6 +145,7 @@ public class InvoicingPipeline implements Serializable {
private TextIO.TypedWrite<BillingEvent, Params> writeDetailReports(
ValueProvider<String> yearMonthProvider) {
return TextIO.<BillingEvent>writeCustomType()
// TODO(larryruili): Replace with billing bucket/yyyy-MM after verifying 2017-12 output.
.to(
InvoicingUtils.makeDestinationFunction(beamBucket + "/results", yearMonthProvider),
InvoicingUtils.makeEmptyDestinationParams(beamBucket + "/results"))

View file

@ -68,7 +68,7 @@ public class InvoicingUtils {
return new Params()
.withBaseFilename(
FileBasedSink.convertToFileResourceIfPossible(
String.format("%s/%s", outputBucket, "failed")));
String.format("%s/%s", outputBucket, "FAILURES")));
}
/**