diff --git a/core/build.gradle b/core/build.gradle index 36b58c519..58680ef0f 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -256,6 +256,7 @@ dependencies { compile deps['org.bouncycastle:bcpg-jdk15on'] testCompile deps['org.bouncycastle:bcpkix-jdk15on'] compile deps['org.bouncycastle:bcprov-jdk15on'] + testCompile deps['com.fasterxml.jackson.core:jackson-databind'] runtime deps['org.glassfish.jaxb:jaxb-runtime'] compile deps['org.hibernate:hibernate-core'] compile deps['org.joda:joda-money'] diff --git a/core/src/main/java/google/registry/tools/RegistryCli.java b/core/src/main/java/google/registry/tools/RegistryCli.java index b268bcaf2..7c2fa3678 100644 --- a/core/src/main/java/google/registry/tools/RegistryCli.java +++ b/core/src/main/java/google/registry/tools/RegistryCli.java @@ -68,8 +68,9 @@ final class RegistryCli implements AutoCloseable, CommandRunner { @Parameter( names = {"--sql_access_info"}, - description = "Name of a file containing space-separated SQL access info used when deploying " - + "Beam pipelines") + description = + "Name of a file containing space-separated SQL access info used when deploying " + + "Beam pipelines") private String sqlAccessInfoFile = null; // Do not make this final - compile-time constant inlining may interfere with JCommander. diff --git a/core/src/test/java/google/registry/beam/TestPipelineExtension.java b/core/src/test/java/google/registry/beam/TestPipelineExtension.java index ac498d090..94441733d 100644 --- a/core/src/test/java/google/registry/beam/TestPipelineExtension.java +++ b/core/src/test/java/google/registry/beam/TestPipelineExtension.java @@ -19,7 +19,7 @@ */ package google.registry.beam; -import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Preconditions.checkState; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -60,9 +60,9 @@ import org.apache.beam.sdk.testing.ValidatesRunner; import org.apache.beam.sdk.transforms.SerializableFunction; import org.apache.beam.sdk.util.common.ReflectHelpers; import org.junit.experimental.categories.Category; -import org.junit.rules.TestRule; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; // NOTE: This file is copied from the Apache Beam distribution so that it can be locally modified to // support JUnit 5. @@ -71,10 +71,6 @@ import org.junit.runners.model.Statement; * A creator of test pipelines that can be used inside of tests that can be configured to run * locally or against a remote pipeline runner. * - *

It is recommended to tag hand-selected tests for this purpose using the {@link - * ValidatesRunner} {@link Category} annotation, as each test run against a pipeline runner will - * utilize resources of that pipeline runner. - * *

In order to run tests on a pipeline runner, the following conditions must be met: * *

*/ -@RunWith(JUnit4.class) -public class LoadDatastoreSnapshotTest { +class LoadDatastoreSnapshotTest { + private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); private static final ImmutableList> ALL_KINDS = @@ -80,12 +78,14 @@ public class LoadDatastoreSnapshotTest { private static final ImmutableSet ALL_KIND_STRS = ALL_KINDS.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet()); - @Rule public final transient TemporaryFolder temporaryFolder = new TemporaryFolder(); + @SuppressWarnings("WeakerAccess") + @TempDir + transient Path tmpDir; - @Rule public final transient InjectRule injectRule = new InjectRule(); + @RegisterExtension final transient InjectRule injectRule = new InjectRule(); - @Rule - public final transient TestPipelineExtension pipeline = + @RegisterExtension + final transient TestPipelineExtension testPipeline = TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); private FakeClock fakeClock; @@ -102,14 +102,14 @@ public class LoadDatastoreSnapshotTest { private transient DateTime contactLastUpdateTime; private transient DateTime domainLastUpdateTime; - @Before - public void beforeEach() throws Exception { + @BeforeEach + void beforeEach() throws Exception { fakeClock = new FakeClock(START_TIME); try (BackupTestStore store = new BackupTestStore(fakeClock)) { injectRule.setStaticField(Ofy.class, "clock", fakeClock); - exportRootDir = temporaryFolder.newFolder(); - commitLogsDir = temporaryFolder.newFolder(); + exportRootDir = Files.createDirectory(tmpDir.resolve("export_root")).toFile(); + commitLogsDir = Files.createDirectory(tmpDir.resolve("commit_logs")).toFile(); Registry registry = newRegistry("tld1", "TLD1"); ContactResource fillerContact = newContactResource("contact_filler"); @@ -154,10 +154,9 @@ public class LoadDatastoreSnapshotTest { } @Test - @Category(NeedsRunner.class) - public void loadDatastoreSnapshot() { + void loadDatastoreSnapshot() { PCollectionTuple snapshot = - pipeline.apply( + testPipeline.apply( Transforms.loadDatastoreSnapshot( exportDir.getAbsolutePath(), commitLogsDir.getAbsolutePath(), @@ -173,6 +172,6 @@ public class LoadDatastoreSnapshotTest { InitSqlTestUtils.assertContainsExactlyElementsIn( snapshot.get(Transforms.createTagForKind("ContactResource")), KV.of(contactLastUpdateTime.getMillis(), dsContact)); - pipeline.run(); + testPipeline.run(); } } diff --git a/core/src/test/java/google/registry/beam/initsql/WriteToSqlTest.java b/core/src/test/java/google/registry/beam/initsql/WriteToSqlTest.java index a1821dd7e..8a98165e2 100644 --- a/core/src/test/java/google/registry/beam/initsql/WriteToSqlTest.java +++ b/core/src/test/java/google/registry/beam/initsql/WriteToSqlTest.java @@ -34,46 +34,48 @@ import google.registry.testing.InjectRule; import java.io.File; import java.io.PrintStream; import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.stream.Collectors; -import org.apache.beam.sdk.testing.NeedsRunner; import org.apache.beam.sdk.transforms.Create; import org.joda.time.DateTime; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.rules.RuleChain; -import org.junit.rules.TemporaryFolder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; /** Unit test for {@link Transforms#writeToSql}. */ -public class WriteToSqlTest implements Serializable { +class WriteToSqlTest implements Serializable { private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); private final FakeClock fakeClock = new FakeClock(START_TIME); - @Rule public final transient InjectRule injectRule = new InjectRule(); + @RegisterExtension final transient InjectRule injectRule = new InjectRule(); - // For use in the RuleChain below. Saves a reference to retrieve Database connection config. - public final transient JpaIntegrationTestRule database = + @RegisterExtension + final transient JpaIntegrationTestRule database = new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationTestRule(); - @Rule - public final transient RuleChain jpaRules = - RuleChain.outerRule(new DatastoreEntityExtension()).around(database); + @RegisterExtension + @Order(value = 1) + final transient DatastoreEntityExtension datastore = new DatastoreEntityExtension(); - @Rule public transient TemporaryFolder temporaryFolder = new TemporaryFolder(); + @SuppressWarnings("WeakerAccess") + @TempDir + transient Path tmpDir; - @Rule - public final transient TestPipelineExtension pipeline = + @RegisterExtension + final transient TestPipelineExtension testPipeline = TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); private ImmutableList contacts; private File credentialFile; - @Before - public void beforeEach() throws Exception { + @BeforeEach + void beforeEach() throws Exception { try (BackupTestStore store = new BackupTestStore(fakeClock)) { injectRule.setStaticField(Ofy.class, "clock", fakeClock); @@ -91,7 +93,7 @@ public class WriteToSqlTest implements Serializable { } contacts = builder.build(); } - credentialFile = temporaryFolder.newFile(); + credentialFile = Files.createFile(tmpDir.resolve("credential.dat")).toFile(); new PrintStream(credentialFile) .printf( "%s %s %s", @@ -102,9 +104,8 @@ public class WriteToSqlTest implements Serializable { } @Test - @Category(NeedsRunner.class) - public void writeToSql_twoWriters() { - pipeline + void writeToSql_twoWriters() { + testPipeline .apply( Create.of( contacts.stream() @@ -121,7 +122,7 @@ public class WriteToSqlTest implements Serializable { .beamJpaModule(new BeamJpaModule(credentialFile.getAbsolutePath())) .build() .localDbJpaTransactionManager())); - pipeline.run().waitUntilFinish(); + testPipeline.run().waitUntilFinish(); ImmutableList sqlContacts = jpaTm().transact(() -> jpaTm().loadAll(ContactResource.class)); // TODO(weiminyu): compare load entities with originals. Note: lastUpdateTimes won't match by diff --git a/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java b/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java index 19ee22ff5..5b584bb6a 100644 --- a/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java +++ b/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java @@ -24,6 +24,8 @@ import google.registry.util.GoogleCredentialsBundle; import google.registry.util.ResourceUtils; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.ZoneId; import java.time.ZonedDateTime; import java.util.Map.Entry; @@ -33,47 +35,46 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; /** Unit tests for {@link InvoicingPipeline}. */ -@RunWith(JUnit4.class) -public class InvoicingPipelineTest { +class InvoicingPipelineTest { private static PipelineOptions pipelineOptions; - @BeforeClass - public static void initializePipelineOptions() { + @BeforeAll + static void beforeAll() { pipelineOptions = PipelineOptionsFactory.create(); pipelineOptions.setRunner(DirectRunner.class); } - @Rule - public final transient TestPipelineExtension p = + @RegisterExtension + final transient TestPipelineExtension testPipeline = TestPipelineExtension.fromOptions(pipelineOptions); - @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); + @SuppressWarnings("WeakerAccess") + @TempDir + transient Path tmpDir; private InvoicingPipeline invoicingPipeline; - @Before - public void initializePipeline() throws IOException { - File beamTempFolder = tempFolder.newFolder(); - String beamTempFolderPath = beamTempFolder.getAbsolutePath(); - invoicingPipeline = new InvoicingPipeline( - "test-project", - beamTempFolderPath, - beamTempFolderPath + "/templates/invoicing", - beamTempFolderPath + "/staging", - tempFolder.getRoot().getAbsolutePath(), - "REG-INV", - GoogleCredentialsBundle.create(GoogleCredentials.create(null)) - ); + @BeforeEach + void beforeEach() throws IOException { + String beamTempFolder = + Files.createDirectory(tmpDir.resolve("beam_temp")).toAbsolutePath().toString(); + invoicingPipeline = + new InvoicingPipeline( + "test-project", + beamTempFolder, + beamTempFolder + "/templates/invoicing", + beamTempFolder + "/staging", + tmpDir.toAbsolutePath().toString(), + "REG-INV", + GoogleCredentialsBundle.create(GoogleCredentials.create(null))); } private ImmutableList getInputEvents() { @@ -189,17 +190,18 @@ public class InvoicingPipelineTest { } @Test - public void testEndToEndPipeline_generatesExpectedFiles() throws Exception { + void testEndToEndPipeline_generatesExpectedFiles() throws Exception { ImmutableList inputRows = getInputEvents(); - PCollection input = p.apply(Create.of(inputRows)); + PCollection input = testPipeline.apply(Create.of(inputRows)); invoicingPipeline.applyTerminalTransforms(input, StaticValueProvider.of("2017-10")); - p.run(); + testPipeline.run(); for (Entry> entry : getExpectedDetailReportMap().entrySet()) { ImmutableList detailReport = resultFileContents(entry.getKey()); assertThat(detailReport.get(0)) - .isEqualTo("id,billingTime,eventTime,registrarId,billingId,poNumber,tld,action," - + "domain,repositoryId,years,currency,amount,flags"); + .isEqualTo( + "id,billingTime,eventTime,registrarId,billingId,poNumber,tld,action," + + "domain,repositoryId,years,currency,amount,flags"); assertThat(detailReport.subList(1, detailReport.size())) .containsExactlyElementsIn(entry.getValue()); } @@ -218,8 +220,7 @@ public class InvoicingPipelineTest { private ImmutableList resultFileContents(String filename) throws Exception { File resultFile = new File( - String.format( - "%s/invoices/2017-10/%s", tempFolder.getRoot().getAbsolutePath(), filename)); + String.format("%s/invoices/2017-10/%s", tmpDir.toAbsolutePath().toString(), filename)); return ImmutableList.copyOf( ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n")); } diff --git a/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java b/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java index 6479a8bcf..04ba955a4 100644 --- a/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java +++ b/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java @@ -38,6 +38,8 @@ import java.io.InputStreamReader; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Comparator; import java.util.function.Supplier; import org.apache.beam.runners.direct.DirectRunner; @@ -56,47 +58,47 @@ import org.joda.time.DateTime; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; /** Unit tests for {@link Spec11Pipeline}. */ -@RunWith(JUnit4.class) -public class Spec11PipelineTest { +class Spec11PipelineTest { private static PipelineOptions pipelineOptions; - @BeforeClass - public static void initializePipelineOptions() { + @BeforeAll + static void beforeAll() { pipelineOptions = PipelineOptionsFactory.create(); pipelineOptions.setRunner(DirectRunner.class); } - @Rule - public final transient TestPipelineExtension p = + @RegisterExtension + final transient TestPipelineExtension testPipeline = TestPipelineExtension.fromOptions(pipelineOptions); - @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); + @SuppressWarnings("WeakerAccess") + @TempDir + Path tmpDir; private final Retrier retrier = new Retrier(new FakeSleeper(new FakeClock(DateTime.parse("2019-07-15TZ"))), 1); private Spec11Pipeline spec11Pipeline; - @Before - public void initializePipeline() throws IOException { - File beamTempFolder = tempFolder.newFolder(); + @BeforeEach + void beforeEach() throws IOException { + String beamTempFolder = + Files.createDirectory(tmpDir.resolve("beam_temp")).toAbsolutePath().toString(); spec11Pipeline = new Spec11Pipeline( "test-project", - beamTempFolder.getAbsolutePath() + "/staging", - beamTempFolder.getAbsolutePath() + "/templates/invoicing", - tempFolder.getRoot().getAbsolutePath(), + beamTempFolder + "/staging", + beamTempFolder + "/templates/invoicing", + tmpDir.toAbsolutePath().toString(), GoogleCredentialsBundle.create(GoogleCredentials.create(null)), retrier); } @@ -130,7 +132,7 @@ public class Spec11PipelineTest { */ @Test @SuppressWarnings("unchecked") - public void testEndToEndPipeline_generatesExpectedFiles() throws Exception { + void testEndToEndPipeline_generatesExpectedFiles() throws Exception { // Establish mocks for testing ImmutableList inputRows = getInputDomains(); CloseableHttpClient httpClient = mock(CloseableHttpClient.class, withSettings().serializable()); @@ -145,9 +147,9 @@ public class Spec11PipelineTest { (Serializable & Supplier) () -> httpClient); // Apply input and evaluation transforms - PCollection input = p.apply(Create.of(inputRows)); + PCollection input = testPipeline.apply(Create.of(inputRows)); spec11Pipeline.evaluateUrlHealth(input, evalFn, StaticValueProvider.of("2018-06-01")); - p.run(); + testPipeline.run(); // Verify header and 4 threat matches for 3 registrars are found ImmutableList generatedReport = resultFileContents(); @@ -295,7 +297,7 @@ public class Spec11PipelineTest { new File( String.format( "%s/icann/spec11/2018-06/SPEC11_MONTHLY_REPORT_2018-06-01", - tempFolder.getRoot().getAbsolutePath())); + tmpDir.toAbsolutePath().toString())); return ImmutableList.copyOf( ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n")); } diff --git a/dependencies.gradle b/dependencies.gradle index 6071fe551..63aed960d 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -130,6 +130,7 @@ ext { 'org.bouncycastle:bcpg-jdk15on:1.61', 'org.bouncycastle:bcpkix-jdk15on:1.61', 'org.bouncycastle:bcprov-jdk15on:1.61', + 'com.fasterxml.jackson.core:jackson-databind:2.9.10', 'org.flywaydb:flyway-core:5.2.4', 'org.glassfish.jaxb:jaxb-runtime:2.3.0', 'org.hamcrest:hamcrest-all:1.3',