mirror of
https://github.com/google/nomulus.git
synced 2025-07-23 19:20:44 +02:00
Make ExportDomainListsAction SQL-aware (#1195)
<!-- Reviewable:start --> This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1195) <!-- Reviewable:end -->
This commit is contained in:
Lai Jiang
GitHub
parent
dad3fcca9a
commit
d113b718d7
2 changed files with 140 additions and 68 deletions
|
|
@ -19,9 +19,10 @@ import static com.google.common.base.Verify.verifyNotNull;
|
|||
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
|
||||
import static google.registry.model.EppResourceUtils.isActive;
|
||||
import static google.registry.model.registry.Registries.getTldsOfType;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
|
||||
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
|
||||
import static google.registry.request.Action.Method.POST;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
import com.google.appengine.tools.cloudstorage.GcsFilename;
|
||||
import com.google.appengine.tools.cloudstorage.RetryParams;
|
||||
|
|
@ -45,12 +46,14 @@ import google.registry.request.Action;
|
|||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.NonFinalForTesting;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.util.List;
|
||||
import java.util.function.Supplier;
|
||||
import javax.inject.Inject;
|
||||
import org.joda.time.DateTime;
|
||||
|
|
@ -70,9 +73,13 @@ public class ExportDomainListsAction implements Runnable {
|
|||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
private static final int MAX_NUM_REDUCE_SHARDS = 100;
|
||||
public static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
@Inject Clock clock;
|
||||
@Inject DriveConnection driveConnection;
|
||||
|
||||
@Inject @Config("domainListsGcsBucket") String gcsBucket;
|
||||
@Inject @Config("gcsBufferSize") int gcsBufferSize;
|
||||
@Inject ExportDomainListsAction() {}
|
||||
|
|
@ -81,15 +88,99 @@ public class ExportDomainListsAction implements Runnable {
|
|||
public void run() {
|
||||
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
|
||||
logger.atInfo().log("Exporting domain lists for tlds %s", realTlds);
|
||||
mrRunner
|
||||
.setJobName("Export domain lists")
|
||||
.setModuleName("backend")
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(DateTime.now(UTC), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
if (tm().isOfy()) {
|
||||
mrRunner
|
||||
.setJobName("Export domain lists")
|
||||
.setModuleName("backend")
|
||||
.setDefaultReduceShards(Math.min(realTlds.size(), MAX_NUM_REDUCE_SHARDS))
|
||||
.runMapreduce(
|
||||
new ExportDomainListsMapper(clock.nowUtc(), realTlds),
|
||||
new ExportDomainListsReducer(gcsBucket, gcsBufferSize),
|
||||
ImmutableList.of(createEntityInput(DomainBase.class)))
|
||||
.sendLinkToMapreduceConsole(response);
|
||||
} else {
|
||||
realTlds.forEach(
|
||||
tld -> {
|
||||
List<String> domains =
|
||||
tm().transact(
|
||||
() ->
|
||||
// Note that if we had "creationTime <= :now" in the condition (not
|
||||
// necessary as there is no pending creation, the order of deletionTime
|
||||
// and creationTime in the query would have been significant and it
|
||||
// should come after deletionTime. When Hibernate substitutes "now" it
|
||||
// will first validate that the **first** field that is to be compared
|
||||
// with it (deletionTime) is assignable from the substituted Java object
|
||||
// (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it
|
||||
// comes first, we will need to substitute "now" with
|
||||
// CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit
|
||||
// strange as the Java object type is clearly incompatible between the
|
||||
// two fields deletionTime (DateTime) and creationTime, yet they are
|
||||
// compared with the same "now". It is actually OK because in the end
|
||||
// Hibernate converts everything to SQL types (and Java field names to
|
||||
// SQL column names) to run the query. Both CreateAutoTimestamp and
|
||||
// DateTime are persisted as timestamp_z in SQL. It is only the
|
||||
// validation that compares the Java types, and only with the first
|
||||
// field that compares with the substituted value.
|
||||
jpaTm()
|
||||
.query(
|
||||
"SELECT fullyQualifiedDomainName FROM Domain "
|
||||
+ "WHERE tld = :tld "
|
||||
+ "AND deletionTime > :now "
|
||||
+ "ORDER by fullyQualifiedDomainName ASC",
|
||||
String.class)
|
||||
.setParameter("tld", tld)
|
||||
.setParameter("now", clock.nowUtc())
|
||||
.getResultList());
|
||||
String domainsList = Joiner.on("\n").join(domains);
|
||||
logger.atInfo().log(
|
||||
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize);
|
||||
exportToDrive(tld, domainsList, driveConnection);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
protected static boolean exportToDrive(
|
||||
String tld, String domains, DriveConnection driveConnection) {
|
||||
verifyNotNull(driveConnection, "Expecting non-null driveConnection");
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
if (registry.getDriveFolderId() == null) {
|
||||
logger.atInfo().log(
|
||||
"Skipping registered domains export for TLD %s because Drive folder isn't specified",
|
||||
tld);
|
||||
} else {
|
||||
String resultMsg =
|
||||
driveConnection.createOrUpdateFile(
|
||||
REGISTERED_DOMAINS_FILENAME,
|
||||
MediaType.PLAIN_TEXT_UTF_8,
|
||||
registry.getDriveFolderId(),
|
||||
domains.getBytes(UTF_8));
|
||||
logger.atInfo().log(
|
||||
"Exporting registered domains succeeded for TLD %s, response was: %s", tld, resultMsg);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to Drive, skipping...", tld);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
protected static boolean exportToGcs(
|
||||
String tld, String domains, String gcsBucket, int gcsBufferSize) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to GCS, skipping...", tld);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static class ExportDomainListsMapper extends Mapper<DomainBase, String, String> {
|
||||
|
|
@ -122,9 +213,6 @@ public class ExportDomainListsAction implements Runnable {
|
|||
private static Supplier<DriveConnection> driveConnectionSupplier =
|
||||
Suppliers.memoize(() -> DaggerDriveModule_DriveComponent.create().driveConnection());
|
||||
|
||||
static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
|
||||
static final MediaType EXPORT_MIME_TYPE = MediaType.PLAIN_TEXT_UTF_8;
|
||||
|
||||
private final String gcsBucket;
|
||||
private final int gcsBufferSize;
|
||||
|
||||
|
|
@ -147,53 +235,21 @@ public class ExportDomainListsAction implements Runnable {
|
|||
driveConnection = driveConnectionSupplier.get();
|
||||
}
|
||||
|
||||
private void exportToDrive(String tld, String domains) {
|
||||
verifyNotNull(driveConnection, "expecting non-null driveConnection");
|
||||
try {
|
||||
Registry registry = Registry.get(tld);
|
||||
if (registry.getDriveFolderId() == null) {
|
||||
logger.atInfo().log(
|
||||
"Skipping registered domains export for TLD %s because Drive folder isn't specified",
|
||||
tld);
|
||||
} else {
|
||||
String resultMsg =
|
||||
driveConnection.createOrUpdateFile(
|
||||
REGISTERED_DOMAINS_FILENAME,
|
||||
EXPORT_MIME_TYPE,
|
||||
registry.getDriveFolderId(),
|
||||
domains.getBytes(UTF_8));
|
||||
logger.atInfo().log(
|
||||
"Exporting registered domains succeeded for TLD %s, response was: %s",
|
||||
tld, resultMsg);
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to Drive", tld);
|
||||
}
|
||||
getContext().incrementCounter("domain lists written out to Drive");
|
||||
}
|
||||
|
||||
private void exportToGcs(String tld, String domains) {
|
||||
GcsFilename filename = new GcsFilename(gcsBucket, tld + ".txt");
|
||||
GcsUtils cloudStorage =
|
||||
new GcsUtils(createGcsService(RetryParams.getDefaultInstance()), gcsBufferSize);
|
||||
try (OutputStream gcsOutput = cloudStorage.openOutputStream(filename);
|
||||
Writer osWriter = new OutputStreamWriter(gcsOutput, UTF_8)) {
|
||||
osWriter.write(domains);
|
||||
} catch (IOException e) {
|
||||
logger.atSevere().withCause(e).log(
|
||||
"Error exporting registered domains for TLD %s to GCS.", tld);
|
||||
}
|
||||
getContext().incrementCounter("domain lists written out to GCS");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void reduce(String tld, ReducerInput<String> fqdns) {
|
||||
ImmutableList<String> domains = ImmutableList.sortedCopyOf(() -> fqdns);
|
||||
String domainsList = Joiner.on('\n').join(domains);
|
||||
logger.atInfo().log("Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
|
||||
exportToGcs(tld, domainsList);
|
||||
exportToDrive(tld, domainsList);
|
||||
if (exportToGcs(tld, domainsList, gcsBucket, gcsBufferSize)) {
|
||||
getContext().incrementCounter("domain lists successful written out to GCS");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to GCS");
|
||||
}
|
||||
if (exportToDrive(tld, domainsList, driveConnection)) {
|
||||
getContext().incrementCounter("domain lists successfully written out to Drive");
|
||||
} else {
|
||||
getContext().incrementCounter("domain lists failed to write out to Drive");
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@ package google.registry.export;
|
|||
|
||||
import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGcsService;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.export.ExportDomainListsAction.ExportDomainListsReducer.EXPORT_MIME_TYPE;
|
||||
import static google.registry.export.ExportDomainListsAction.ExportDomainListsReducer.REGISTERED_DOMAINS_FILENAME;
|
||||
import static google.registry.export.ExportDomainListsAction.REGISTERED_DOMAINS_FILENAME;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
|
||||
|
|
@ -34,25 +33,40 @@ import com.google.appengine.tools.cloudstorage.GcsFilename;
|
|||
import com.google.appengine.tools.cloudstorage.GcsService;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.appengine.tools.cloudstorage.ListResult;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.export.ExportDomainListsAction.ExportDomainListsReducer;
|
||||
import google.registry.model.ofy.Ofy;
|
||||
import google.registry.model.registry.Registry;
|
||||
import google.registry.model.registry.Registry.TldType;
|
||||
import google.registry.storage.drive.DriveConnection;
|
||||
import google.registry.testing.DualDatabaseTest;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.InjectExtension;
|
||||
import google.registry.testing.TestOfyAndSql;
|
||||
import google.registry.testing.TestOfyOnly;
|
||||
import google.registry.testing.mapreduce.MapreduceTestCase;
|
||||
import java.io.FileNotFoundException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.Order;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
/** Unit tests for {@link ExportDomainListsAction}. */
|
||||
@DualDatabaseTest
|
||||
class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAction> {
|
||||
|
||||
private GcsService gcsService;
|
||||
private DriveConnection driveConnection = mock(DriveConnection.class);
|
||||
private ArgumentCaptor<byte[]> bytesExportedToDrive = ArgumentCaptor.forClass(byte[].class);
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final FakeClock clock = new FakeClock(DateTime.parse("2020-02-02T02:02:02Z"));
|
||||
|
||||
@Order(Order.DEFAULT - 1)
|
||||
@RegisterExtension
|
||||
public final InjectExtension inject =
|
||||
new InjectExtension().withStaticFieldOverride(Ofy.class, "clock", clock);
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
|
|
@ -68,10 +82,12 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
action.response = response;
|
||||
action.gcsBucket = "outputbucket";
|
||||
action.gcsBufferSize = 500;
|
||||
action.clock = clock;
|
||||
action.driveConnection = driveConnection;
|
||||
gcsService = createGcsService();
|
||||
}
|
||||
|
||||
private void runMapreduce() throws Exception {
|
||||
private void runAction() throws Exception {
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce");
|
||||
}
|
||||
|
|
@ -80,27 +96,27 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
verify(driveConnection)
|
||||
.createOrUpdateFile(
|
||||
eq(REGISTERED_DOMAINS_FILENAME),
|
||||
eq(EXPORT_MIME_TYPE),
|
||||
eq(MediaType.PLAIN_TEXT_UTF_8),
|
||||
eq(folderId),
|
||||
bytesExportedToDrive.capture());
|
||||
assertThat(new String(bytesExportedToDrive.getValue(), UTF_8)).isEqualTo(domains);
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestOfyOnly
|
||||
void test_writesLinkToMapreduceConsoleToResponse() throws Exception {
|
||||
runMapreduce();
|
||||
runAction();
|
||||
assertThat(response.getPayload())
|
||||
.startsWith(
|
||||
"Mapreduce console: https://backend-dot-projectid.appspot.com"
|
||||
+ "/_ah/pipeline/status.html?root=");
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestOfyAndSql
|
||||
void test_outputsOnlyActiveDomains() throws Exception {
|
||||
persistActiveDomain("onetwo.tld");
|
||||
persistActiveDomain("rudnitzky.tld");
|
||||
persistDeletedDomain("mortuary.tld", DateTime.parse("2001-03-14T10:11:12Z"));
|
||||
runMapreduce();
|
||||
runAction();
|
||||
GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8);
|
||||
// Check that it only contains the active domains, not the dead one.
|
||||
|
|
@ -109,12 +125,12 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
verifyNoMoreInteractions(driveConnection);
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestOfyAndSql
|
||||
void test_outputsOnlyDomainsOnRealTlds() throws Exception {
|
||||
persistActiveDomain("onetwo.tld");
|
||||
persistActiveDomain("rudnitzky.tld");
|
||||
persistActiveDomain("wontgo.testtld");
|
||||
runMapreduce();
|
||||
runAction();
|
||||
GcsFilename existingFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, existingFile), UTF_8).trim();
|
||||
// Check that it only contains the domains on the real TLD, and not the test one.
|
||||
|
|
@ -130,7 +146,7 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
verifyNoMoreInteractions(driveConnection);
|
||||
}
|
||||
|
||||
@Test
|
||||
@TestOfyAndSql
|
||||
void test_outputsDomainsFromDifferentTldsToMultipleFiles() throws Exception {
|
||||
createTld("tldtwo");
|
||||
persistResource(Registry.get("tldtwo").asBuilder().setDriveFolderId("hooray").build());
|
||||
|
|
@ -143,7 +159,7 @@ class ExportDomainListsActionTest extends MapreduceTestCase<ExportDomainListsAct
|
|||
persistActiveDomain("santa.tldtwo");
|
||||
persistActiveDomain("buddy.tldtwo");
|
||||
persistActiveDomain("cupid.tldthree");
|
||||
runMapreduce();
|
||||
runAction();
|
||||
GcsFilename firstTldFile = new GcsFilename("outputbucket", "tld.txt");
|
||||
String tlds = new String(readGcsFile(gcsService, firstTldFile), UTF_8).trim();
|
||||
assertThat(tlds).isEqualTo("dasher.tld\nprancer.tld");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue