Upload to GCS before uploading to FTP

Currently we encode and upload the deposite to GCS and the FTP server at the
same time. This makes debugging harder as there are many possible points of
failure, some of which are external and some internal.

In this CL we start by encoding + uploading the deposit to GCS, and once
that's done we copy the data from GCS to the FTP server. This will (hopefully)
allow us to distinguish between errors on the FTP server and errors with the
GCS connection.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=199643208

java/google/registry/rde/EscrowTaskRunner.java

@@ -87,14 +87,15 @@ class EscrowTaskRunner {
       final Duration interval) {
     Callable<Void> lockRunner =
         () -> {
-          logger.atInfo().log("TLD: %s", registry.getTld());
           DateTime startOfToday = clock.nowUtc().withTimeAtStartOfDay();
           Cursor cursor = ofy().load().key(Cursor.createKey(cursorType, registry)).now();
+          logger.atInfo().log(
+              "TLD: %s, cursorType: %s cursor: %s", registry.getTld(), cursorType, cursor);
           final DateTime nextRequiredRun = (cursor == null ? startOfToday : cursor.getCursorTime());
           if (nextRequiredRun.isAfter(startOfToday)) {
             throw new NoContentException("Already completed");
           }
-          logger.atInfo().log("Cursor: %s", nextRequiredRun);
+          logger.atInfo().log("CursorTime: %s", nextRequiredRun);
           task.runWithLock(nextRequiredRun);
           ofy()
               .transact(

java/google/registry/rde/RdeStagingReducer.java

@@ -132,6 +132,8 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
     final int revision =
         Optional.ofNullable(key.revision())
             .orElse(RdeRevision.getNextRevision(tld, watermark, mode));
+    logger.atInfo().log(
+        "tld=%s watermark=%s mode=%s key=%s revision=%s", tld, watermark, mode, key, revision);
     String id = RdeUtil.timestampToId(watermark);
     String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
     if (key.manual()) {
@@ -245,7 +247,8 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
                   key);
               ofy().save().entity(Cursor.create(key.cursor(), newPosition, registry)).now();
               logger.atInfo().log(
-                  "Rolled forward %s on %s cursor to %s", key.cursor(), tld, newPosition);
+                  "Rolled forward %s on %s cursor to %s. Watermark=%s, mode=%s, revision=%s",
+                  key.cursor(), tld, newPosition, watermark, mode, revision);
               RdeRevision.saveRevision(tld, watermark, mode, revision);
               if (mode == RdeMode.FULL) {
                 taskQueueUtils.enqueue(

java/google/registry/rde/RdeUploadAction.java

@@ -24,7 +24,6 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.rde.RdeMode.FULL;
 import static google.registry.request.Action.Method.POST;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Arrays.asList;
 
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.appengine.tools.cloudstorage.GcsFilename;
@@ -53,7 +52,6 @@ import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.TeeOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -132,7 +130,8 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
 
   @Override
   public void runWithLock(final DateTime watermark) throws Exception {
-    logger.atInfo().log("Verifying readiness to upload the RDE deposit.");
+    logger.atInfo().log(
+        "Verifying readiness to upload the RDE deposit for tld=%s, watermark=%s.", tld, watermark);
     DateTime stagingCursorTime = getCursorTimeOrStartOfTime(
         ofy().load().key(Cursor.createKey(CursorType.RDE_STAGING, Registry.get(tld))).now());
     if (!stagingCursorTime.isAfter(watermark)) {
@@ -205,19 +204,17 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
   protected void upload(
       GcsFilename xmlFile, long xmlLength, DateTime watermark, String name) throws Exception {
     logger.atInfo().log("Uploading XML file '%s' to remote path '%s'.", xmlFile, uploadUrl);
+    byte[] signature;
+    String rydeFilename = name + ".ryde";
+    String sigFilename = name + ".sig";
+    GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename);
+    // Encode and save the files to GCS
     try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile);
         Ghostryde.Decryptor decryptor = ghostryde.openDecryptor(gcsInput, stagingDecryptionKey);
         Ghostryde.Decompressor decompressor = ghostryde.openDecompressor(decryptor);
         Ghostryde.Input xmlInput = ghostryde.openInput(decompressor)) {
-      try (JSchSshSession session = jschSshSessionFactory.create(lazyJsch.get(), uploadUrl);
+      try (OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
-          JSchSftpChannel ftpChan = session.openSftpChannel()) {
+          RydePgpSigningOutputStream signer = pgpSigningFactory.create(gcsOutput, signingKey)) {
-        byte[] signature;
-        String rydeFilename = name + ".ryde";
-        GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename);
-        try (OutputStream ftpOutput = ftpChan.get().put(rydeFilename, OVERWRITE);
-            OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
-            TeeOutputStream teeOutput = new TeeOutputStream(asList(ftpOutput, gcsOutput));
-            RydePgpSigningOutputStream signer = pgpSigningFactory.create(teeOutput, signingKey)) {
         try (OutputStream encryptLayer = pgpEncryptionFactory.create(signer, receiverKey);
             OutputStream kompressor = pgpCompressionFactory.create(encryptLayer);
             OutputStream fileLayer = pgpFileFactory.create(kompressor, watermark, name + ".tar");
@@ -226,13 +223,27 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
           ByteStreams.copy(xmlInput, tarLayer);
         }
         signature = signer.getSignature();
-          logger.atInfo().log("uploaded %,d bytes: %s.ryde", signer.getBytesWritten(), name);
+        logger.atInfo().log(
+            "uploaded %,d bytes to gcs bucket %s, file %s",
+            signer.getBytesWritten(), bucket, rydeFilename);
       }
-        String sigFilename = name + ".sig";
       gcsUtils.createFromBytes(new GcsFilename(bucket, sigFilename), signature);
-        ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
+      logger.atInfo().log(
-        logger.atInfo().log("uploaded %,d bytes: %s.sig", signature.length, name);
+          "uploaded %,d bytes to gcs bucket %s, file %s", signature.length, bucket, sigFilename);
     }
+    // Copy the file from GCS to the FTP server
+    try (JSchSshSession session = jschSshSessionFactory.create(lazyJsch.get(), uploadUrl);
+        JSchSftpChannel ftpChan = session.openSftpChannel()) {
+
+      try (OutputStream ftpOutput = ftpChan.get().put(rydeFilename, OVERWRITE);
+          InputStream gcsInput = gcsUtils.openInputStream(rydeGcsFilename)) {
+        long bytesCopied = ByteStreams.copy(gcsInput, ftpOutput);
+        logger.atInfo().log(
+            "uploaded %,d bytes to ftp path %s, file %s", bytesCopied, uploadUrl, rydeFilename);
+      }
+      ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
+      logger.atInfo().log(
+          "uploaded %,d bytes to ftp path %s, file %s", signature.length, uploadUrl, sigFilename);
     }
   }
 

java/google/registry/util/TeeOutputStream.java

@@ -1,67 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.util;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.collect.ImmutableList;
-import java.io.IOException;
-import java.io.OutputStream;
-import javax.annotation.WillNotClose;
-
-/**
- * {@link OutputStream} delegate that writes simultaneously to multiple other output streams.
-  */
-public final class TeeOutputStream extends OutputStream {
-
-  private final ImmutableList<? extends OutputStream> outputs;
-  private boolean isClosed;
-
-  public TeeOutputStream(@WillNotClose Iterable<? extends OutputStream> outputs) {
-    this.outputs = ImmutableList.copyOf(outputs);
-    checkArgument(!this.outputs.isEmpty(), "must provide at least one output stream");
-  }
-
-  /** @see java.io.OutputStream#write(int) */
-  @Override
-  public void write(int b) throws IOException {
-    checkState(!isClosed, "outputstream closed");
-    for (OutputStream out : outputs) {
-      out.write(b);
-    }
-  }
-
-  /** @see #write(byte[], int, int) */
-  @Override
-  public void write(byte[] b) throws IOException {
-    this.write(b, 0, b.length);
-  }
-
-  /** @see java.io.OutputStream#write(byte[], int, int) */
-  @Override
-  public void write(byte[] b, int off, int len) throws IOException {
-    checkState(!isClosed, "outputstream closed");
-    for (OutputStream out : outputs) {
-      out.write(b, off, len);
-    }
-  }
-
-  /** Closes the stream.  Any calls to a {@code write()} method after this will throw. */
-  @Override
-  public void close() throws IOException {
-    isClosed = true;
-  }
-}

javatests/google/registry/util/TeeOutputStreamTest.java

@@ -1,86 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.util;
-
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.testing.JUnitBackports.assertThrows;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.util.Arrays.asList;
-
-import com.google.common.collect.ImmutableSet;
-import java.io.ByteArrayOutputStream;
-import java.io.OutputStream;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Unit tests for {@link TeeOutputStream}. */
-@RunWith(JUnit4.class)
-public class TeeOutputStreamTest {
-  private final ByteArrayOutputStream outputA = new ByteArrayOutputStream();
-  private final ByteArrayOutputStream outputB = new ByteArrayOutputStream();
-  private final ByteArrayOutputStream outputC = new ByteArrayOutputStream();
-
-  @Test
-  public void testWrite_writesToMultipleStreams() throws Exception {
-    // Write shared data using the tee output stream.
-    try (OutputStream tee =
-        new TeeOutputStream(asList(outputA, outputB, outputC))) {
-      tee.write("hello ".getBytes(UTF_8));
-      tee.write("hello world!".getBytes(UTF_8), 6, 5);
-      tee.write('!');
-    }
-    // Write some more data to the different streams - they should not have been closed.
-    outputA.write("a".getBytes(UTF_8));
-    outputB.write("b".getBytes(UTF_8));
-    outputC.write("c".getBytes(UTF_8));
-    // Check the results.
-    assertThat(outputA.toString()).isEqualTo("hello world!a");
-    assertThat(outputB.toString()).isEqualTo("hello world!b");
-    assertThat(outputC.toString()).isEqualTo("hello world!c");
-  }
-
-  @Test
-  @SuppressWarnings("resource")
-  public void testConstructor_failsWithEmptyIterable() {
-    assertThrows(IllegalArgumentException.class, () -> new TeeOutputStream(ImmutableSet.of()));
-  }
-
-  @Test
-  public void testWriteInteger_failsAfterClose() throws Exception {
-    OutputStream tee = new TeeOutputStream(asList(outputA));
-    tee.close();
-    IllegalStateException thrown = assertThrows(IllegalStateException.class, () -> tee.write(1));
-    assertThat(thrown).hasMessageThat().contains("outputstream closed");
-  }
-
-  @Test
-  public void testWriteByteArray_failsAfterClose() throws Exception {
-    OutputStream tee = new TeeOutputStream(asList(outputA));
-    tee.close();
-    IllegalStateException thrown =
-        assertThrows(IllegalStateException.class, () -> tee.write("hello".getBytes(UTF_8)));
-    assertThat(thrown).hasMessageThat().contains("outputstream closed");
-  }
-
-  @Test
-  public void testWriteByteSubarray_failsAfterClose() throws Exception {
-    OutputStream tee = new TeeOutputStream(asList(outputA));
-    tee.close();
-    IllegalStateException thrown =
-        assertThrows(IllegalStateException.class, () -> tee.write("hello".getBytes(UTF_8), 1, 3));
-    assertThat(thrown).hasMessageThat().contains("outputstream closed");
-  }
-}