Move invoice generation to billing bucket and improve emailing

This moves the new pipeline's invoice generation to the billing bucket, under the 'invoices/yyyy-MM' subdirectory.

This also changes the invoice e-mail to use a multipart message that attaches the invoice to the e-mail, to guarantee the correct MIME type and download.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=181746191

javatests/google/registry/beam/InvoicingPipelineTest.java

@@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import google.registry.util.ResourceUtils;
 import java.io.File;
+import java.io.IOException;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.Map.Entry;
@@ -56,10 +57,15 @@ public class InvoicingPipelineTest {
   private InvoicingPipeline invoicingPipeline;
 
   @Before
-  public void initializePipeline() {
+  public void initializePipeline() throws IOException {
     invoicingPipeline = new InvoicingPipeline();
     invoicingPipeline.projectId = "test-project";
-    invoicingPipeline.beamBucket = tempFolder.getRoot().getAbsolutePath();
+    File beamTempFolder = tempFolder.newFolder();
+    invoicingPipeline.beamBucketUrl = beamTempFolder.getAbsolutePath();
+    invoicingPipeline.invoiceStagingUrl = beamTempFolder.getAbsolutePath() + "/staging";
+    invoicingPipeline.invoiceTemplateUrl =
+        beamTempFolder.getAbsolutePath() + "/templates/invoicing";
+    invoicingPipeline.billingBucketUrl = tempFolder.getRoot().getAbsolutePath();
   }
 
   private ImmutableList<BillingEvent> getInputEvents() {
@@ -180,7 +186,9 @@ public class InvoicingPipelineTest {
   /** Returns the text contents of a file under the beamBucket/results directory. */
   private ImmutableList<String> resultFileContents(String filename) throws Exception {
     File resultFile =
-        new File(String.format("%s/results/%s", tempFolder.getRoot().getAbsolutePath(), filename));
+        new File(
+            String.format(
+                "%s/invoices/2017-10/%s", tempFolder.getRoot().getAbsolutePath(), filename));
     return ImmutableList.copyOf(
         ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n"));
   }

javatests/google/registry/beam/InvoicingUtilsTest.java

@@ -49,7 +49,7 @@ public class InvoicingUtilsTest {
                 .withSuffix(".csv")
                 .withBaseFilename(
                     FileBasedSink.convertToFileResourceIfPossible(
-                        "my/directory/invoice_details_2017-10_registrar_tld")));
+                        "my/directory/2017-10/invoice_details_2017-10_registrar_tld")));
   }
 
   @Test

javatests/google/registry/billing/BillingEmailUtilsTest.java

@@ -15,15 +15,29 @@
 package google.registry.billing;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
+import static google.registry.testing.JUnitBackports.expectThrows;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.appengine.tools.cloudstorage.GcsFilename;
 import com.google.common.collect.ImmutableList;
+import google.registry.gcs.GcsUtils;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeSleeper;
+import google.registry.util.Retrier;
 import google.registry.util.SendEmailService;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Properties;
+import javax.mail.BodyPart;
 import javax.mail.Message;
 import javax.mail.MessagingException;
+import javax.mail.Multipart;
 import javax.mail.Session;
 import javax.mail.internet.InternetAddress;
 import javax.mail.internet.MimeMessage;
@@ -32,45 +46,125 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
+import org.mockito.ArgumentCaptor;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 
 /** Unit tests for {@link BillingEmailUtils}. */
 @RunWith(JUnit4.class)
 public class BillingEmailUtilsTest {
 
+  private static final int RETRY_COUNT = 2;
+
   private SendEmailService emailService;
-  private Message msg;
   private BillingEmailUtils emailUtils;
+  private GcsUtils gcsUtils;
+  private ArgumentCaptor<Message> msgCaptor;
 
   @Before
   public void setUp() {
-    msg = new MimeMessage(Session.getDefaultInstance(new Properties(), null));
     emailService = mock(SendEmailService.class);
-    when(emailService.createMessage()).thenReturn(msg);
+    when(emailService.createMessage())
+        .thenReturn(new MimeMessage(Session.getDefaultInstance(new Properties(), null)));
+    gcsUtils = mock(GcsUtils.class);
+    when(gcsUtils.openInputStream(new GcsFilename("test-bucket", "results/CRR-INV-2017-10.csv")))
+        .thenReturn(
+            new ByteArrayInputStream("test,data\nhello,world".getBytes(StandardCharsets.UTF_8)));
+    msgCaptor = ArgumentCaptor.forClass(Message.class);
 
     emailUtils =
         new BillingEmailUtils(
             emailService,
             new YearMonth(2017, 10),
             "my-sender@test.com",
+            "my-receiver@test.com",
             ImmutableList.of("hello@world.com", "hola@mundo.com"),
-            "gs://test-bucket");
+            "test-bucket",
+            "results/",
+            gcsUtils,
+            new Retrier(new FakeSleeper(new FakeClock()), RETRY_COUNT));
   }
 
   @Test
-  public void testSuccess_sendsEmail() throws MessagingException, IOException {
-    emailUtils.emailInvoiceLink();
-    assertThat(msg.getFrom()).hasLength(1);
-    assertThat(msg.getFrom()[0])
-        .isEqualTo(new InternetAddress("my-sender@test.com"));
-    assertThat(msg.getAllRecipients())
+  public void testSuccess_emailOverallInvoice() throws MessagingException, IOException {
+    emailUtils.emailOverallInvoice();
+    // We inspect individual parameters because Message doesn't implement equals().
+    verify(emailService).sendMessage(msgCaptor.capture());
+    Message expectedMsg = msgCaptor.getValue();
+    assertThat(expectedMsg.getFrom())
+        .asList()
+        .containsExactly(new InternetAddress("my-sender@test.com"));
+    assertThat(expectedMsg.getAllRecipients())
         .asList()
         .containsExactly(
             new InternetAddress("hello@world.com"), new InternetAddress("hola@mundo.com"));
-    assertThat(msg.getSubject()).isEqualTo("Domain Registry invoice data 2017-10");
+    assertThat(expectedMsg.getSubject()).isEqualTo("Domain Registry invoice data 2017-10");
+    assertThat(expectedMsg.getContent()).isInstanceOf(Multipart.class);
+    Multipart contents = (Multipart) expectedMsg.getContent();
+    assertThat(contents.getCount()).isEqualTo(2);
+    assertThat(contents.getBodyPart(0)).isInstanceOf(BodyPart.class);
+    BodyPart textPart = contents.getBodyPart(0);
+    assertThat(textPart.getContentType()).isEqualTo("text/plain; charset=us-ascii");
+    assertThat(textPart.getContent().toString())
+        .isEqualTo("Attached is the 2017-10 invoice for the domain registry.");
+    assertThat(contents.getBodyPart(1)).isInstanceOf(BodyPart.class);
+    BodyPart attachmentPart = contents.getBodyPart(1);
+    assertThat(attachmentPart.getContentType())
+        .isEqualTo("text/csv; charset=utf-8; name=CRR-INV-2017-10.csv");
+    assertThat(attachmentPart.getContent().toString()).isEqualTo("test,data\nhello,world");
+  }
+
+  @Test
+  public void testFailure_tooManyRetries_emailsAlert() throws MessagingException, IOException {
+    // This message throws whenever it tries to set content, to force the overall invoice to fail.
+    Message throwingMessage = mock(Message.class);
+    doThrow(new MessagingException("expected"))
+        .when(throwingMessage)
+        .setContent(any(Multipart.class));
+    when(emailService.createMessage()).thenAnswer(
+        new Answer<Message>() {
+          private int count = 0;
+
+          @Override
+          public Message answer(InvocationOnMock invocation) throws Throwable {
+            // Once we've failed the retry limit for the original invoice, return a normal message
+            // so we can properly check its contents.
+            if (count < RETRY_COUNT) {
+              count++;
+              return throwingMessage;
+            } else if (count == RETRY_COUNT) {
+              return new MimeMessage(Session.getDefaultInstance(new Properties(), null));
+            } else {
+              assertWithMessage("Attempted to generate too many messages!").fail();
+              return null;
+            }
+          }
+        }
+    );
+    RuntimeException thrown =
+        expectThrows(RuntimeException.class, () -> emailUtils.emailOverallInvoice());
+    assertThat(thrown).hasMessageThat().isEqualTo("javax.mail.MessagingException: expected");
+    // Verify we sent an e-mail alert
+    verify(emailService).sendMessage(msgCaptor.capture());
+    validateAlertMessage(msgCaptor.getValue(), "Emailing invoice failed due to expected");
+  }
+
+  @Test
+  public void testSuccess_sendAlertEmail() throws MessagingException, IOException {
+    emailUtils.sendAlertEmail("Alert!");
+    verify(emailService).sendMessage(msgCaptor.capture());
+    validateAlertMessage(msgCaptor.getValue(), "Alert!");
+  }
+
+  private void validateAlertMessage(Message msg, String body)
+      throws MessagingException, IOException {
+    assertThat(msg.getFrom()).hasLength(1);
+    assertThat(msg.getFrom()[0]).isEqualTo(new InternetAddress("my-sender@test.com"));
+    assertThat(msg.getAllRecipients())
+        .asList()
+        .containsExactly(new InternetAddress("my-receiver@test.com"));
+    assertThat(msg.getSubject()).isEqualTo("Billing Pipeline Alert: 2017-10");
     assertThat(msg.getContentType()).isEqualTo("text/plain");
-    assertThat(msg.getContent().toString())
-        .isEqualTo(
-            "Link to invoice on GCS:\n"
-                + "https://storage.cloud.google.com/test-bucket/results/CRR-INV-2017-10.csv");
+    assertThat(msg.getContent().toString()).isEqualTo(body);
   }
 }

javatests/google/registry/billing/CopyDetailReportsActionTest.java

@@ -15,6 +15,7 @@
 package google.registry.billing;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth.assertWithMessage;
 import static google.registry.testing.DatastoreHelper.loadRegistrar;
 import static google.registry.testing.DatastoreHelper.persistResource;
 import static google.registry.testing.GcsTestingUtils.writeGcsFile;
@@ -61,6 +62,7 @@ public class CopyDetailReportsActionTest {
 
   private FakeResponse response;
   private DriveConnection driveConnection;
+  private BillingEmailUtils emailUtils;
   private CopyDetailReportsAction action;
 
   @Before
@@ -68,14 +70,16 @@ public class CopyDetailReportsActionTest {
     persistResource(loadRegistrar("TheRegistrar").asBuilder().setDriveFolderId("0B-12345").build());
     response = new FakeResponse();
     driveConnection = mock(DriveConnection.class);
+    emailUtils = mock(BillingEmailUtils.class);
     action =
         new CopyDetailReportsAction(
-            "gs://test-bucket",
+            "test-bucket",
             "results/",
             driveConnection,
             gcsUtils,
             new Retrier(new FakeSleeper(new FakeClock()), 3),
-            response);
+            response,
+            emailUtils);
   }
 
   @Test
@@ -156,6 +160,31 @@ public class CopyDetailReportsActionTest {
     assertThat(response.getPayload()).isEqualTo("Copied detail reports.");
   }
 
+  @Test
+  public void testFail_tooManyFailures_sendsAlertEmail() throws IOException {
+    writeGcsFile(
+        gcsService,
+        new GcsFilename("test-bucket", "results/invoice_details_2017-10_TheRegistrar_hello.csv"),
+        "hola,mundo\n3,4".getBytes(UTF_8));
+    when(driveConnection.createFile(any(), any(), any(), any()))
+        .thenThrow(new IOException("expected"));
+
+    try {
+      action.run();
+      assertWithMessage("Expected a runtime exception to be thrown!").fail();
+    } catch (RuntimeException e) {
+      assertThat(e).hasMessageThat().isEqualTo("java.io.IOException: expected");
+    }
+    verify(driveConnection, times(3))
+        .createFile(
+            "invoice_details_2017-10_TheRegistrar_hello.csv",
+            MediaType.CSV_UTF_8,
+            "0B-12345",
+            "hola,mundo\n3,4".getBytes(UTF_8));
+    verify(emailUtils).sendAlertEmail("Warning: CopyDetailReportsAction failed.\nEncountered: "
+        + "expected on file: invoice_details_2017-10_TheRegistrar_hello.csv");
+  }
+
   @Test
   public void testFail_registrarDoesntExist_doesntCopy() throws IOException {
     writeGcsFile(

javatests/google/registry/billing/GenerateInvoicesActionTest.java

@@ -53,6 +53,7 @@ public class GenerateInvoicesActionTest {
   private Templates templates;
   private Launch launch;
   private FakeResponse response;
+  private BillingEmailUtils emailUtils;
   private GenerateInvoicesAction action;
 
   @Before
@@ -61,6 +62,7 @@ public class GenerateInvoicesActionTest {
     projects = mock(Projects.class);
     templates = mock(Templates.class);
     launch = mock(Launch.class);
+    emailUtils = mock(BillingEmailUtils.class);
     when(dataflow.projects()).thenReturn(projects);
     when(projects.templates()).thenReturn(templates);
     when(templates.launch(any(String.class), any(LaunchTemplateParameters.class)))
@@ -72,8 +74,15 @@ public class GenerateInvoicesActionTest {
     job.setId("12345");
     when(launch.execute()).thenReturn(new LaunchTemplateResponse().setJob(job));
 
-    action = new GenerateInvoicesAction(
-        "test-project", "gs://test-project-beam", new YearMonth(2017, 10), dataflow, response);
+    action =
+        new GenerateInvoicesAction(
+            "test-project",
+            "gs://test-project-beam",
+            "gs://test-project-beam/templates/invoicing",
+            new YearMonth(2017, 10),
+            dataflow,
+            response,
+            emailUtils);
   }
 
   @Test
@@ -96,7 +105,8 @@ public class GenerateInvoicesActionTest {
         new TaskMatcher()
             .url("/_dr/task/publishInvoices")
             .method("POST")
-            .param("jobId", "12345");
+            .param("jobId", "12345")
+            .param("yearMonth", "2017-10");
     assertTasksEnqueued("billing", matcher);
   }
 
@@ -106,5 +116,6 @@ public class GenerateInvoicesActionTest {
     action.run();
     assertThat(response.getStatus()).isEqualTo(500);
     assertThat(response.getPayload()).isEqualTo("Template launch failed: expected");
+    verify(emailUtils).sendAlertEmail("Template Launch failed due to expected");
   }
 }

javatests/google/registry/billing/PublishInvoicesActionTest.java

@@ -34,6 +34,7 @@ import google.registry.testing.AppEngineRule;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.io.IOException;
+import org.joda.time.YearMonth;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -70,7 +71,8 @@ public class PublishInvoicesActionTest {
     emailUtils = mock(BillingEmailUtils.class);
     response = new FakeResponse();
     uploadAction =
-        new PublishInvoicesAction("test-project", "12345", emailUtils, dataflow, response);
+        new PublishInvoicesAction(
+            "test-project", "12345", emailUtils, dataflow, response, new YearMonth(2017, 10));
   }
 
   @Test
@@ -78,12 +80,12 @@ public class PublishInvoicesActionTest {
     expectedJob.setCurrentState("JOB_STATE_DONE");
     uploadAction.run();
     assertThat(response.getStatus()).isEqualTo(SC_OK);
-    verify(emailUtils).emailInvoiceLink();
+    verify(emailUtils).emailOverallInvoice();
     TaskMatcher matcher =
         new TaskMatcher()
             .url("/_dr/task/copyDetailReports")
             .method("POST")
-            .param("directoryPrefix", "results/");
+            .param("yearMonth", "2017-10");
     assertTasksEnqueued("retryable-cron-tasks", matcher);
   }
 
@@ -92,6 +94,7 @@ public class PublishInvoicesActionTest {
     expectedJob.setCurrentState("JOB_STATE_FAILED");
     uploadAction.run();
     assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
+    verify(emailUtils).sendAlertEmail("Dataflow job 12345 ended in status failure.");
   }
 
   @Test
@@ -108,5 +111,6 @@ public class PublishInvoicesActionTest {
     assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
     assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("Template launch failed: expected");
+    verify(emailUtils).sendAlertEmail("Publish action failed due to expected");
   }
 }