zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-15 08:57:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

Upload to GCS before uploading to FTP

Browse source 
Currently we encode and upload the deposite to GCS and the FTP server at the
same time. This makes debugging harder as there are many possible points of
failure, some of which are external and some internal.

In this CL we start by encoding + uploading the deposit to GCS, and once
that's done we copy the data from GCS to the FTP server. This will (hopefully)
allow us to distinguish between errors on the FTP server and errors with the
GCS connection.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=199643208
This commit is contained in:
guyben 2018-06-07 09:03:52 -07:00 • committed by Ben McIlwain
parent 228e4f6c95
commit ea08661598
5 changed files with 43 additions and 181 deletions
Download patch file Download diff file Expand all files Collapse all files

5
java/google/registry/rde/EscrowTaskRunner.java
View file

@ -87,14 +87,15 @@ class EscrowTaskRunner {
final Duration interval) {
Callable<Void> lockRunner =
() -> {
logger.atInfo().log("TLD: %s", registry.getTld());
DateTime startOfToday = clock.nowUtc().withTimeAtStartOfDay();
Cursor cursor = ofy().load().key(Cursor.createKey(cursorType, registry)).now();
logger.atInfo().log(
"TLD: %s, cursorType: %s cursor: %s", registry.getTld(), cursorType, cursor);
final DateTime nextRequiredRun = (cursor == null ? startOfToday : cursor.getCursorTime());
if (nextRequiredRun.isAfter(startOfToday)) {
throw new NoContentException("Already completed");
}
logger.atInfo().log("Cursor: %s", nextRequiredRun);
logger.atInfo().log("CursorTime: %s", nextRequiredRun);
task.runWithLock(nextRequiredRun);
ofy()
.transact(

5
java/google/registry/rde/RdeStagingReducer.java
View file

@ -132,6 +132,8 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
final int revision =
Optional.ofNullable(key.revision())
.orElse(RdeRevision.getNextRevision(tld, watermark, mode));
logger.atInfo().log(
"tld=%s watermark=%s mode=%s key=%s revision=%s", tld, watermark, mode, key, revision);
String id = RdeUtil.timestampToId(watermark);
String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
if (key.manual()) {
@ -245,7 +247,8 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
key);
ofy().save().entity(Cursor.create(key.cursor(), newPosition, registry)).now();
logger.atInfo().log(
"Rolled forward %s on %s cursor to %s", key.cursor(), tld, newPosition);
"Rolled forward %s on %s cursor to %s. Watermark=%s, mode=%s, revision=%s",
key.cursor(), tld, newPosition, watermark, mode, revision);
RdeRevision.saveRevision(tld, watermark, mode, revision);
if (mode == RdeMode.FULL) {
taskQueueUtils.enqueue(

43
java/google/registry/rde/RdeUploadAction.java
View file

@ -24,7 +24,6 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.model.rde.RdeMode.FULL;
import static google.registry.request.Action.Method.POST;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.tools.cloudstorage.GcsFilename;
@ -53,7 +52,6 @@ import google.registry.request.auth.Auth;
import google.registry.util.Clock;
import google.registry.util.Retrier;
import google.registry.util.TaskQueueUtils;
import google.registry.util.TeeOutputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -132,7 +130,8 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
@Override
public void runWithLock(final DateTime watermark) throws Exception {
logger.atInfo().log("Verifying readiness to upload the RDE deposit.");
logger.atInfo().log(
"Verifying readiness to upload the RDE deposit for tld=%s, watermark=%s.", tld, watermark);
DateTime stagingCursorTime = getCursorTimeOrStartOfTime(
ofy().load().key(Cursor.createKey(CursorType.RDE_STAGING, Registry.get(tld))).now());
if (!stagingCursorTime.isAfter(watermark)) {
@ -205,19 +204,17 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
protected void upload(
GcsFilename xmlFile, long xmlLength, DateTime watermark, String name) throws Exception {
logger.atInfo().log("Uploading XML file '%s' to remote path '%s'.", xmlFile, uploadUrl);
byte[] signature;
String rydeFilename = name + ".ryde";
String sigFilename = name + ".sig";
GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename);
// Encode and save the files to GCS
try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile);
Ghostryde.Decryptor decryptor = ghostryde.openDecryptor(gcsInput, stagingDecryptionKey);
Ghostryde.Decompressor decompressor = ghostryde.openDecompressor(decryptor);
Ghostryde.Input xmlInput = ghostryde.openInput(decompressor)) {
try (JSchSshSession session = jschSshSessionFactory.create(lazyJsch.get(), uploadUrl);
JSchSftpChannel ftpChan = session.openSftpChannel()) {
byte[] signature;
String rydeFilename = name + ".ryde";
GcsFilename rydeGcsFilename = new GcsFilename(bucket, rydeFilename);
try (OutputStream ftpOutput = ftpChan.get().put(rydeFilename, OVERWRITE);
OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
TeeOutputStream teeOutput = new TeeOutputStream(asList(ftpOutput, gcsOutput));
RydePgpSigningOutputStream signer = pgpSigningFactory.create(teeOutput, signingKey)) {
try (OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
RydePgpSigningOutputStream signer = pgpSigningFactory.create(gcsOutput, signingKey)) {
try (OutputStream encryptLayer = pgpEncryptionFactory.create(signer, receiverKey);
OutputStream kompressor = pgpCompressionFactory.create(encryptLayer);
OutputStream fileLayer = pgpFileFactory.create(kompressor, watermark, name + ".tar");
@ -226,13 +223,27 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
ByteStreams.copy(xmlInput, tarLayer);
}
signature = signer.getSignature();
logger.atInfo().log("uploaded %,d bytes: %s.ryde", signer.getBytesWritten(), name);
logger.atInfo().log(
"uploaded %,d bytes to gcs bucket %s, file %s",
signer.getBytesWritten(), bucket, rydeFilename);
}
String sigFilename = name + ".sig";
gcsUtils.createFromBytes(new GcsFilename(bucket, sigFilename), signature);
ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
logger.atInfo().log("uploaded %,d bytes: %s.sig", signature.length, name);
logger.atInfo().log(
"uploaded %,d bytes to gcs bucket %s, file %s", signature.length, bucket, sigFilename);
}
// Copy the file from GCS to the FTP server
try (JSchSshSession session = jschSshSessionFactory.create(lazyJsch.get(), uploadUrl);
JSchSftpChannel ftpChan = session.openSftpChannel()) {
try (OutputStream ftpOutput = ftpChan.get().put(rydeFilename, OVERWRITE);
InputStream gcsInput = gcsUtils.openInputStream(rydeGcsFilename)) {
long bytesCopied = ByteStreams.copy(gcsInput, ftpOutput);
logger.atInfo().log(
"uploaded %,d bytes to ftp path %s, file %s", bytesCopied, uploadUrl, rydeFilename);
}
ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
logger.atInfo().log(
"uploaded %,d bytes to ftp path %s, file %s", signature.length, uploadUrl, sigFilename);
}
}

67
java/google/registry/util/TeeOutputStream.java
View file

@ -1,67 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.util;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.io.OutputStream;
import javax.annotation.WillNotClose;
/**
* {@link OutputStream} delegate that writes simultaneously to multiple other output streams.
*/
public final class TeeOutputStream extends OutputStream {
private final ImmutableList<? extends OutputStream> outputs;
private boolean isClosed;
public TeeOutputStream(@WillNotClose Iterable<? extends OutputStream> outputs) {
this.outputs = ImmutableList.copyOf(outputs);
checkArgument(!this.outputs.isEmpty(), "must provide at least one output stream");
}
/** @see java.io.OutputStream#write(int) */
@Override
public void write(int b) throws IOException {
checkState(!isClosed, "outputstream closed");
for (OutputStream out : outputs) {
out.write(b);
}
}
/** @see #write(byte[], int, int) */
@Override
public void write(byte[] b) throws IOException {
this.write(b, 0, b.length);
}
/** @see java.io.OutputStream#write(byte[], int, int) */
@Override
public void write(byte[] b, int off, int len) throws IOException {
checkState(!isClosed, "outputstream closed");
for (OutputStream out : outputs) {
out.write(b, off, len);
}
}
/** Closes the stream. Any calls to a {@code write()} method after this will throw. */
@Override
public void close() throws IOException {
isClosed = true;
}
}

86
javatests/google/registry/util/TeeOutputStreamTest.java
View file

@ -1,86 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.util;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.testing.JUnitBackports.assertThrows;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
import com.google.common.collect.ImmutableSet;
import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Unit tests for {@link TeeOutputStream}. */
@RunWith(JUnit4.class)
public class TeeOutputStreamTest {
private final ByteArrayOutputStream outputA = new ByteArrayOutputStream();
private final ByteArrayOutputStream outputB = new ByteArrayOutputStream();
private final ByteArrayOutputStream outputC = new ByteArrayOutputStream();
@Test
public void testWrite_writesToMultipleStreams() throws Exception {
// Write shared data using the tee output stream.
try (OutputStream tee =
new TeeOutputStream(asList(outputA, outputB, outputC))) {
tee.write("hello ".getBytes(UTF_8));
tee.write("hello world!".getBytes(UTF_8), 6, 5);
tee.write('!');
}
// Write some more data to the different streams - they should not have been closed.
outputA.write("a".getBytes(UTF_8));
outputB.write("b".getBytes(UTF_8));
outputC.write("c".getBytes(UTF_8));
// Check the results.
assertThat(outputA.toString()).isEqualTo("hello world!a");
assertThat(outputB.toString()).isEqualTo("hello world!b");
assertThat(outputC.toString()).isEqualTo("hello world!c");
}
@Test
@SuppressWarnings("resource")
public void testConstructor_failsWithEmptyIterable() {
assertThrows(IllegalArgumentException.class, () -> new TeeOutputStream(ImmutableSet.of()));
}
@Test
public void testWriteInteger_failsAfterClose() throws Exception {
OutputStream tee = new TeeOutputStream(asList(outputA));
tee.close();
IllegalStateException thrown = assertThrows(IllegalStateException.class, () -> tee.write(1));
assertThat(thrown).hasMessageThat().contains("outputstream closed");
}
@Test
public void testWriteByteArray_failsAfterClose() throws Exception {
OutputStream tee = new TeeOutputStream(asList(outputA));
tee.close();
IllegalStateException thrown =
assertThrows(IllegalStateException.class, () -> tee.write("hello".getBytes(UTF_8)));
assertThat(thrown).hasMessageThat().contains("outputstream closed");
}
@Test
public void testWriteByteSubarray_failsAfterClose() throws Exception {
OutputStream tee = new TeeOutputStream(asList(outputA));
tee.close();
IllegalStateException thrown =
assertThrows(IllegalStateException.class, () -> tee.write("hello".getBytes(UTF_8), 1, 3));
assertThat(thrown).hasMessageThat().contains("outputstream closed");
}
}