Add publish functionality to billing pipeline

This closes the end-to-end billing pipeline, allowing us to share generated detail reports with registrars via Drive and e-mail the invoicing team a link to the generated invoice.

This also factors out the email configs from ICANN reporting into the common 'misc' config, since we'll likely need alert e-mails for future periodic tasks.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=180805972

javatests/google/registry/beam/InvoicingUtilsTest.java

@@ -58,7 +58,7 @@ public class InvoicingUtilsTest {
         .isEqualTo(
             new Params()
                 .withBaseFilename(
-                    FileBasedSink.convertToFileResourceIfPossible("my/directory/failed")));
+                    FileBasedSink.convertToFileResourceIfPossible("my/directory/FAILURES")));
   }
 
   /** Asserts that the instantiated sql template matches a golden expected file. */

javatests/google/registry/billing/BUILD

@@ -12,8 +12,13 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//java/google/registry/billing",
+        "//java/google/registry/gcs",
+        "//java/google/registry/storage/drive",
+        "//java/google/registry/util",
         "//javatests/google/registry/testing",
         "@com_google_apis_google_api_services_dataflow",
+        "@com_google_appengine_api_1_0_sdk",
+        "@com_google_appengine_tools_appengine_gcs_client",
         "@com_google_dagger",
         "@com_google_guava",
         "@com_google_truth",

javatests/google/registry/billing/BillingEmailUtilsTest.java

@@ -0,0 +1,76 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.billing;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import google.registry.util.SendEmailService;
+import java.io.IOException;
+import java.util.Properties;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.Session;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+import org.joda.time.YearMonth;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link BillingEmailUtils}. */
+@RunWith(JUnit4.class)
+public class BillingEmailUtilsTest {
+
+  private SendEmailService emailService;
+  private Message msg;
+  private BillingEmailUtils emailUtils;
+
+  @Before
+  public void setUp() {
+    msg = new MimeMessage(Session.getDefaultInstance(new Properties(), null));
+    emailService = mock(SendEmailService.class);
+    when(emailService.createMessage()).thenReturn(msg);
+
+    emailUtils =
+        new BillingEmailUtils(
+            emailService,
+            new YearMonth(2017, 10),
+            "my-sender@test.com",
+            ImmutableList.of("hello@world.com", "hola@mundo.com"),
+            "gs://test-bucket");
+  }
+
+  @Test
+  public void testSuccess_sendsEmail() throws MessagingException, IOException {
+    emailUtils.emailInvoiceLink();
+    assertThat(msg.getFrom()).hasLength(1);
+    assertThat(msg.getFrom()[0])
+        .isEqualTo(new InternetAddress("my-sender@test.com"));
+    assertThat(msg.getAllRecipients())
+        .asList()
+        .containsExactly(
+            new InternetAddress("hello@world.com"), new InternetAddress("hola@mundo.com"));
+    assertThat(msg.getSubject()).isEqualTo("Domain Registry invoice data 2017-10");
+    assertThat(msg.getContentType()).isEqualTo("text/plain");
+    assertThat(msg.getContent().toString())
+        .isEqualTo(
+            "Link to invoice on GCS:\n"
+                + "https://storage.cloud.google.com/test-bucket/results/CRR-INV-2017-10.csv");
+  }
+}

javatests/google/registry/billing/CopyDetailReportsActionTest.java

@@ -0,0 +1,180 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.billing;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.DatastoreHelper.loadRegistrar;
+import static google.registry.testing.DatastoreHelper.persistResource;
+import static google.registry.testing.GcsTestingUtils.writeGcsFile;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.common.net.MediaType;
+import google.registry.gcs.GcsUtils;
+import google.registry.storage.drive.DriveConnection;
+import google.registry.testing.AppEngineRule;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeSleeper;
+import google.registry.util.Retrier;
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link CopyDetailReportsAction}. */
+@RunWith(JUnit4.class)
+public class CopyDetailReportsActionTest {
+
+  @Rule
+  public final AppEngineRule appEngine = AppEngineRule.builder()
+      .withDatastore()
+      .build();
+
+
+  private final GcsService gcsService = GcsServiceFactory.createGcsService();
+  private final GcsUtils gcsUtils = new GcsUtils(gcsService, 1024);
+
+  private FakeResponse response;
+  private DriveConnection driveConnection;
+  private CopyDetailReportsAction action;
+
+  @Before
+  public void setUp() {
+    persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId("0B-12345").build());
+    response = new FakeResponse();
+    driveConnection = mock(DriveConnection.class);
+    action =
+        new CopyDetailReportsAction(
+            "gs://test-bucket",
+            "results/",
+            driveConnection,
+            gcsUtils,
+            new Retrier(new FakeSleeper(new FakeClock()), 3),
+            response);
+  }
+
+  @Test
+  public void testSuccess() throws IOException {
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_test.csv"),
+        "hello,world\n1,2".getBytes(UTF_8));
+
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
+        "hola,mundo\n3,4".getBytes(UTF_8));
+
+    action.run();
+    verify(driveConnection)
+        .createFile(
+            "invoice_details_2017-10_TheRegistrar_test.csv",
+            MediaType.CSV_UTF_8,
+            "0B-12345",
+            "hello,world\n1,2".getBytes(UTF_8));
+
+    verify(driveConnection)
+        .createFile(
+            "invoice_details_2017-10_TheRegistrar_hello.csv",
+            MediaType.CSV_UTF_8,
+            "0B-12345",
+            "hola,mundo\n3,4".getBytes(UTF_8));
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
+    assertThat(response.getPayload()).isEqualTo("Copied detail reports.");
+  }
+
+  @Test
+  public void testSuccess_nonDetailReportFiles_notSent() throws IOException{
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
+        "hola,mundo\n3,4".getBytes(UTF_8));
+
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("test-bucket", "results/not_a_detail_report_2017-10_TheRegistrar_test.csv"),
+        "hello,world\n1,2".getBytes(UTF_8));
+    action.run();
+    verify(driveConnection)
+        .createFile(
+            "invoice_details_2017-10_TheRegistrar_hello.csv",
+            MediaType.CSV_UTF_8,
+            "0B-12345",
+            "hola,mundo\n3,4".getBytes(UTF_8));
+    // Verify we didn't copy the non-detail report file.
+    verifyNoMoreInteractions(driveConnection);
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
+    assertThat(response.getPayload()).isEqualTo("Copied detail reports.");
+  }
+
+  @Test
+  public void testSuccess_transientIOException_retries() throws IOException {
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
+        "hola,mundo\n3,4".getBytes(UTF_8));
+    when(driveConnection.createFile(any(), any(), any(), any()))
+        .thenThrow(new IOException("expected"))
+        .thenReturn("success");
+
+    action.run();
+    verify(driveConnection, times(2))
+        .createFile(
+            "invoice_details_2017-10_TheRegistrar_hello.csv",
+            MediaType.CSV_UTF_8,
+            "0B-12345",
+            "hola,mundo\n3,4".getBytes(UTF_8));
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
+    assertThat(response.getPayload()).isEqualTo("Copied detail reports.");
+  }
+
+  @Test
+  public void testFail_registrarDoesntExist_doesntCopy() throws IOException {
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("test-bucket", "results/invoice_details_2017-10_notExistent_hello.csv"),
+        "hola,mundo\n3,4".getBytes(UTF_8));
+    action.run();
+    verifyZeroInteractions(driveConnection);
+  }
+
+  @Test
+  public void testFail_noRegistrarFolderId_doesntCopy() throws IOException {
+    persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId(null).build());
+    writeGcsFile(
+        gcsService,
+        new GcsFilename(
+            "test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
+        "hola,mundo\n3,4".getBytes(UTF_8));
+    action.run();
+    verifyZeroInteractions(driveConnection);
+  }
+}

javatests/google/registry/billing/GenerateInvoicesActionTest.java

@@ -45,33 +45,35 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class GenerateInvoicesActionTest {
 
-  Dataflow dataflow = mock(Dataflow.class);
-  Projects projects = mock(Projects.class);
-  Templates templates = mock(Templates.class);
-  Launch launch = mock(Launch.class);
-  GenerateInvoicesAction action;
-  FakeResponse response = new FakeResponse();
-
   @Rule
   public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
 
+  private Dataflow dataflow;
+  private Projects projects;
+  private Templates templates;
+  private Launch launch;
+  private FakeResponse response;
+  private GenerateInvoicesAction action;
+
   @Before
-  public void initializeObjects() throws Exception {
+  public void setUp() throws IOException {
+    dataflow = mock(Dataflow.class);
+    projects = mock(Projects.class);
+    templates = mock(Templates.class);
+    launch = mock(Launch.class);
     when(dataflow.projects()).thenReturn(projects);
     when(projects.templates()).thenReturn(templates);
     when(templates.launch(any(String.class), any(LaunchTemplateParameters.class)))
         .thenReturn(launch);
     when(launch.setGcsPath(any(String.class))).thenReturn(launch);
+
+    response = new FakeResponse();
     Job job = new Job();
     job.setId("12345");
     when(launch.execute()).thenReturn(new LaunchTemplateResponse().setJob(job));
 
-    action = new GenerateInvoicesAction();
-    action.dataflow = dataflow;
-    action.response = response;
-    action.projectId = "test-project";
-    action.beamBucketUrl = "gs://test-project-beam";
-    action.yearMonth = new YearMonth(2017, 10);
+    action = new GenerateInvoicesAction(
+        "test-project", "gs://test-project-beam", new YearMonth(2017, 10), dataflow, response);
   }
 
   @Test
@@ -99,7 +101,7 @@ public class GenerateInvoicesActionTest {
   }
 
   @Test
-  public void testCaughtIOException() throws Exception {
+  public void testCaughtIOException() throws IOException {
     when(launch.execute()).thenThrow(new IOException("expected"));
     action.run();
     assertThat(response.getStatus()).isEqualTo(500);

javatests/google/registry/billing/PublishInvoicesActionTest.java

@@ -15,11 +15,13 @@
 package google.registry.billing;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NOT_MODIFIED;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import com.google.api.services.dataflow.Dataflow;
@@ -28,9 +30,12 @@ import com.google.api.services.dataflow.Dataflow.Projects.Jobs;
 import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Get;
 import com.google.api.services.dataflow.model.Job;
 import com.google.common.net.MediaType;
+import google.registry.testing.AppEngineRule;
 import google.registry.testing.FakeResponse;
+import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.io.IOException;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -38,36 +43,48 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class PublishInvoicesActionTest {
 
-  private final Dataflow dataflow = mock(Dataflow.class);
-  private final Projects projects = mock(Projects.class);
-  private final Jobs jobs = mock(Jobs.class);
-  private final Get get = mock(Get.class);
+  private Dataflow dataflow;
+  private Projects projects;
+  private Jobs jobs;
+  private Get get;
+  private BillingEmailUtils emailUtils;
 
   private Job expectedJob;
   private FakeResponse response;
   private PublishInvoicesAction uploadAction;
 
+  @Rule
+  public final AppEngineRule appEngine = AppEngineRule.builder().withTaskQueue().build();
+
   @Before
-  public void initializeObjects() throws Exception {
+  public void setUp() throws IOException {
+    dataflow = mock(Dataflow.class);
+    projects = mock(Projects.class);
+    jobs = mock(Jobs.class);
+    get = mock(Get.class);
     when(dataflow.projects()).thenReturn(projects);
     when(projects.jobs()).thenReturn(jobs);
     when(jobs.get("test-project", "12345")).thenReturn(get);
     expectedJob = new Job();
     when(get.execute()).thenReturn(expectedJob);
-
-    uploadAction = new PublishInvoicesAction();
-    uploadAction.projectId = "test-project";
-    uploadAction.jobId = "12345";
-    uploadAction.dataflow = dataflow;
+    emailUtils = mock(BillingEmailUtils.class);
     response = new FakeResponse();
-    uploadAction.response = response;
+    uploadAction =
+        new PublishInvoicesAction("test-project", "12345", emailUtils, dataflow, response);
   }
 
   @Test
-  public void testJobDone_returnsSuccess() {
+  public void testJobDone_enqueuesCopyAction_emailsResults() throws Exception {
     expectedJob.setCurrentState("JOB_STATE_DONE");
     uploadAction.run();
     assertThat(response.getStatus()).isEqualTo(SC_OK);
+    verify(emailUtils).emailInvoiceLink();
+    TaskMatcher matcher =
+        new TaskMatcher()
+            .url("/_dr/task/copyDetailReports")
+            .method("POST")
+            .param("directoryPrefix", "results/");
+    assertTasksEnqueued("retryable-cron-tasks", matcher);
   }
 
   @Test
@@ -85,7 +102,7 @@ public class PublishInvoicesActionTest {
   }
 
   @Test
-  public void testIOException_returnsFailureMessage() throws Exception {
+  public void testIOException_returnsFailureMessage() throws IOException {
     when(get.execute()).thenThrow(new IOException("expected"));
     uploadAction.run();
     assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);

javatests/google/registry/module/backend/testdata/backend_routing.txt

@@ -6,6 +6,7 @@ PATH                                   CLASS                              METHOD
 /_dr/dnsRefresh                        RefreshDnsAction                   GET      y  INTERNAL     APP IGNORED
 /_dr/task/brdaCopy                     BrdaCopyAction                     POST     y  INTERNAL     APP IGNORED
 /_dr/task/checkSnapshot                CheckSnapshotAction                POST,GET y  INTERNAL     APP IGNORED
+/_dr/task/copyDetailReports            CopyDetailReportsAction            POST     n  INTERNAL,API APP ADMIN
 /_dr/task/deleteContactsAndHosts       DeleteContactsAndHostsAction       GET      n  INTERNAL     APP IGNORED
 /_dr/task/deleteOldCommitLogs          DeleteOldCommitLogsAction          GET      n  INTERNAL     APP IGNORED
 /_dr/task/deleteProberData             DeleteProberDataAction             POST     n  INTERNAL     APP IGNORED