mirror of
https://github.com/google/nomulus.git
synced 2025-07-07 03:33:28 +02:00
Migrate the billing pipeline to flex template (#1100)
This is similar to the migration of the spec11 pipeline in #1073. Also removed a few Dagger providers that are no longer needed. TESTED=tested the dataflow job on alpha. <!-- Reviewable:start --> --- This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1100) <!-- Reviewable:end -->
This commit is contained in:
Lai Jiang
GitHub
parent
09b6e300fc
commit
217b37b9d5
26 changed files with 554 additions and 807 deletions
|
|
@ -18,52 +18,20 @@ import static com.google.common.truth.Truth.assertThat;
|
|||
import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
|
||||
import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
|
||||
import static org.apache.http.HttpStatus.SC_OK;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.dataflow.Dataflow;
|
||||
import com.google.api.services.dataflow.model.Job;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
|
||||
import google.registry.beam.BeamActionTestBase;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
/** Unit tests for {@link WipeoutDatastoreAction}. */
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class WipeOutDatastoreActionTest {
|
||||
|
||||
@Mock private Dataflow dataflow;
|
||||
@Mock private Dataflow.Projects projects;
|
||||
@Mock private Dataflow.Projects.Locations locations;
|
||||
@Mock private Dataflow.Projects.Locations.FlexTemplates flexTemplates;
|
||||
@Mock private Dataflow.Projects.Locations.FlexTemplates.Launch launch;
|
||||
private LaunchFlexTemplateResponse launchResponse =
|
||||
new LaunchFlexTemplateResponse().setJob(new Job());
|
||||
class WipeOutDatastoreActionTest extends BeamActionTestBase {
|
||||
|
||||
private final FakeClock clock = new FakeClock();
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
lenient().when(dataflow.projects()).thenReturn(projects);
|
||||
lenient().when(projects.locations()).thenReturn(locations);
|
||||
lenient().when(locations.flexTemplates()).thenReturn(flexTemplates);
|
||||
lenient()
|
||||
.when(flexTemplates.launch(anyString(), anyString(), any(LaunchFlexTemplateRequest.class)))
|
||||
.thenReturn(launch);
|
||||
lenient().when(launch.execute()).thenReturn(launchResponse);
|
||||
}
|
||||
|
||||
@Test
|
||||
void run_projectNotAllowed() {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.dataflow.Dataflow;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations.FlexTemplates;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations.FlexTemplates.Launch;
|
||||
import com.google.api.services.dataflow.model.Job;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
|
||||
/** Base class for all actions that launches a Dataflow Flex template. */
|
||||
@MockitoSettings(strictness = Strictness.STRICT_STUBS)
|
||||
public abstract class BeamActionTestBase {
|
||||
|
||||
protected FakeResponse response = new FakeResponse();
|
||||
protected Dataflow dataflow = mock(Dataflow.class);
|
||||
private Projects projects = mock(Projects.class);
|
||||
private Locations locations = mock(Locations.class);
|
||||
private FlexTemplates templates = mock(FlexTemplates.class);
|
||||
protected Launch launch = mock(Launch.class);
|
||||
private LaunchFlexTemplateResponse launchResponse =
|
||||
new LaunchFlexTemplateResponse().setJob(new Job().setId("jobid"));
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws Exception {
|
||||
when(dataflow.projects()).thenReturn(projects);
|
||||
when(projects.locations()).thenReturn(locations);
|
||||
when(locations.flexTemplates()).thenReturn(templates);
|
||||
when(templates.launch(anyString(), anyString(), any(LaunchFlexTemplateRequest.class)))
|
||||
.thenReturn(launch);
|
||||
when(launch.execute()).thenReturn(launchResponse);
|
||||
}
|
||||
}
|
||||
|
|
@ -16,26 +16,21 @@ package google.registry.beam.invoicing;
|
|||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import com.google.auth.oauth2.GoogleCredentials;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.beam.TestPipelineExtension;
|
||||
import google.registry.util.GoogleCredentialsBundle;
|
||||
import google.registry.testing.TestDataHelper;
|
||||
import google.registry.util.ResourceUtils;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.Map.Entry;
|
||||
import org.apache.beam.runners.direct.DirectRunner;
|
||||
import org.apache.beam.sdk.options.PipelineOptions;
|
||||
import org.apache.beam.sdk.coders.SerializableCoder;
|
||||
import org.apache.beam.sdk.options.PipelineOptionsFactory;
|
||||
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
|
||||
import org.apache.beam.sdk.transforms.Create;
|
||||
import org.apache.beam.sdk.values.PCollection;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
|
@ -44,160 +39,168 @@ import org.junit.jupiter.api.io.TempDir;
|
|||
/** Unit tests for {@link InvoicingPipeline}. */
|
||||
class InvoicingPipelineTest {
|
||||
|
||||
private static PipelineOptions pipelineOptions;
|
||||
private static final String BILLING_BUCKET_URL = "billing_bucket";
|
||||
private static final String YEAR_MONTH = "2017-10";
|
||||
private static final String INVOICE_FILE_PREFIX = "REG-INV";
|
||||
|
||||
@BeforeAll
|
||||
static void beforeAll() {
|
||||
pipelineOptions = PipelineOptionsFactory.create();
|
||||
pipelineOptions.setRunner(DirectRunner.class);
|
||||
}
|
||||
private static final ImmutableList<BillingEvent> INPUT_EVENTS =
|
||||
ImmutableList.of(
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
"234",
|
||||
"",
|
||||
"test",
|
||||
"RENEW",
|
||||
"mydomain.test",
|
||||
"REPO-ID",
|
||||
3,
|
||||
"USD",
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
"234",
|
||||
"",
|
||||
"test",
|
||||
"RENEW",
|
||||
"mydomain2.test",
|
||||
"REPO-ID",
|
||||
3,
|
||||
"USD",
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 2, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 9, 29, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
"234",
|
||||
"",
|
||||
"hello",
|
||||
"CREATE",
|
||||
"mydomain3.hello",
|
||||
"REPO-ID",
|
||||
5,
|
||||
"JPY",
|
||||
70.75,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"bestdomains",
|
||||
"456",
|
||||
"116688",
|
||||
"test",
|
||||
"RENEW",
|
||||
"mydomain4.test",
|
||||
"REPO-ID",
|
||||
1,
|
||||
"USD",
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"anotherRegistrar",
|
||||
"789",
|
||||
"",
|
||||
"test",
|
||||
"CREATE",
|
||||
"mydomain5.test",
|
||||
"REPO-ID",
|
||||
1,
|
||||
"USD",
|
||||
0,
|
||||
"SUNRISE ANCHOR_TENANT"));
|
||||
|
||||
private static final ImmutableMap<String, ImmutableList<String>> EXPECTED_DETAILED_REPORT_MAP =
|
||||
ImmutableMap.of(
|
||||
"invoice_details_2017-10_theRegistrar_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,RENEW,mydomain2.test,REPO-ID,3,USD,20.50,",
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,RENEW,mydomain.test,REPO-ID,3,USD,20.50,"),
|
||||
"invoice_details_2017-10_theRegistrar_hello.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-02 00:00:00 UTC,2017-09-29 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "hello,CREATE,mydomain3.hello,REPO-ID,5,JPY,70.75,"),
|
||||
"invoice_details_2017-10_bestdomains_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,bestdomains,456,116688,"
|
||||
+ "test,RENEW,mydomain4.test,REPO-ID,1,USD,20.50,"),
|
||||
"invoice_details_2017-10_anotherRegistrar_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,anotherRegistrar,789,,"
|
||||
+ "test,CREATE,mydomain5.test,REPO-ID,1,USD,0.00,SUNRISE ANCHOR_TENANT"));
|
||||
|
||||
private static final ImmutableList<String> EXPECTED_INVOICE_OUTPUT =
|
||||
ImmutableList.of(
|
||||
"2017-10-01,2020-09-30,234,41.00,USD,10125,1,PURCHASE,theRegistrar - test,2,"
|
||||
+ "RENEW | TLD: test | TERM: 3-year,20.50,USD,",
|
||||
"2017-10-01,2022-09-30,234,70.75,JPY,10125,1,PURCHASE,theRegistrar - hello,1,"
|
||||
+ "CREATE | TLD: hello | TERM: 5-year,70.75,JPY,",
|
||||
"2017-10-01,2018-09-30,456,20.50,USD,10125,1,PURCHASE,bestdomains - test,1,"
|
||||
+ "RENEW | TLD: test | TERM: 1-year,20.50,USD,116688");
|
||||
|
||||
@RegisterExtension
|
||||
final transient TestPipelineExtension testPipeline =
|
||||
TestPipelineExtension.fromOptions(pipelineOptions);
|
||||
final TestPipelineExtension pipeline =
|
||||
TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);
|
||||
|
||||
@SuppressWarnings("WeakerAccess")
|
||||
@TempDir
|
||||
transient Path tmpDir;
|
||||
@TempDir Path tmpDir;
|
||||
|
||||
private InvoicingPipeline invoicingPipeline;
|
||||
private final InvoicingPipelineOptions options =
|
||||
PipelineOptionsFactory.create().as(InvoicingPipelineOptions.class);
|
||||
|
||||
private File billingBucketUrl;
|
||||
private PCollection<BillingEvent> billingEvents;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws IOException {
|
||||
String beamTempFolder =
|
||||
Files.createDirectory(tmpDir.resolve("beam_temp")).toAbsolutePath().toString();
|
||||
invoicingPipeline =
|
||||
new InvoicingPipeline(
|
||||
"test-project",
|
||||
"region",
|
||||
beamTempFolder,
|
||||
beamTempFolder + "/templates/invoicing",
|
||||
beamTempFolder + "/staging",
|
||||
tmpDir.toAbsolutePath().toString(),
|
||||
"REG-INV",
|
||||
GoogleCredentialsBundle.create(GoogleCredentials.create(null)));
|
||||
}
|
||||
|
||||
private ImmutableList<BillingEvent> getInputEvents() {
|
||||
return ImmutableList.of(
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
"234",
|
||||
"",
|
||||
"test",
|
||||
"RENEW",
|
||||
"mydomain.test",
|
||||
"REPO-ID",
|
||||
3,
|
||||
"USD",
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
"234",
|
||||
"",
|
||||
"test",
|
||||
"RENEW",
|
||||
"mydomain2.test",
|
||||
"REPO-ID",
|
||||
3,
|
||||
"USD",
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 2, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 9, 29, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"theRegistrar",
|
||||
"234",
|
||||
"",
|
||||
"hello",
|
||||
"CREATE",
|
||||
"mydomain3.hello",
|
||||
"REPO-ID",
|
||||
5,
|
||||
"JPY",
|
||||
70.75,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"bestdomains",
|
||||
"456",
|
||||
"116688",
|
||||
"test",
|
||||
"RENEW",
|
||||
"mydomain4.test",
|
||||
"REPO-ID",
|
||||
1,
|
||||
"USD",
|
||||
20.5,
|
||||
""),
|
||||
BillingEvent.create(
|
||||
1,
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
ZonedDateTime.of(2017, 10, 4, 0, 0, 0, 0, ZoneId.of("UTC")),
|
||||
"anotherRegistrar",
|
||||
"789",
|
||||
"",
|
||||
"test",
|
||||
"CREATE",
|
||||
"mydomain5.test",
|
||||
"REPO-ID",
|
||||
1,
|
||||
"USD",
|
||||
0,
|
||||
"SUNRISE ANCHOR_TENANT"));
|
||||
}
|
||||
|
||||
/** Returns a map from filename to expected contents for detail reports. */
|
||||
private ImmutableMap<String, ImmutableList<String>> getExpectedDetailReportMap() {
|
||||
return ImmutableMap.of(
|
||||
"invoice_details_2017-10_theRegistrar_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,RENEW,mydomain2.test,REPO-ID,3,USD,20.50,",
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "test,RENEW,mydomain.test,REPO-ID,3,USD,20.50,"),
|
||||
"invoice_details_2017-10_theRegistrar_hello.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-02 00:00:00 UTC,2017-09-29 00:00:00 UTC,theRegistrar,234,,"
|
||||
+ "hello,CREATE,mydomain3.hello,REPO-ID,5,JPY,70.75,"),
|
||||
"invoice_details_2017-10_bestdomains_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,bestdomains,456,116688,"
|
||||
+ "test,RENEW,mydomain4.test,REPO-ID,1,USD,20.50,"),
|
||||
"invoice_details_2017-10_anotherRegistrar_test.csv",
|
||||
ImmutableList.of(
|
||||
"1,2017-10-04 00:00:00 UTC,2017-10-04 00:00:00 UTC,anotherRegistrar,789,,"
|
||||
+ "test,CREATE,mydomain5.test,REPO-ID,1,USD,0.00,SUNRISE ANCHOR_TENANT"));
|
||||
}
|
||||
|
||||
private ImmutableList<String> getExpectedInvoiceOutput() {
|
||||
return ImmutableList.of(
|
||||
"2017-10-01,2020-09-30,234,41.00,USD,10125,1,PURCHASE,theRegistrar - test,2,"
|
||||
+ "RENEW | TLD: test | TERM: 3-year,20.50,USD,",
|
||||
"2017-10-01,2022-09-30,234,70.75,JPY,10125,1,PURCHASE,theRegistrar - hello,1,"
|
||||
+ "CREATE | TLD: hello | TERM: 5-year,70.75,JPY,",
|
||||
"2017-10-01,2018-09-30,456,20.50,USD,10125,1,PURCHASE,bestdomains - test,1,"
|
||||
+ "RENEW | TLD: test | TERM: 1-year,20.50,USD,116688");
|
||||
void beforeEach() throws Exception {
|
||||
billingBucketUrl = Files.createDirectory(tmpDir.resolve(BILLING_BUCKET_URL)).toFile();
|
||||
options.setBillingBucketUrl(billingBucketUrl.getAbsolutePath());
|
||||
options.setYearMonth(YEAR_MONTH);
|
||||
options.setInvoiceFilePrefix(INVOICE_FILE_PREFIX);
|
||||
billingEvents =
|
||||
pipeline.apply(Create.of(INPUT_EVENTS).withCoder(SerializableCoder.of(BillingEvent.class)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testEndToEndPipeline_generatesExpectedFiles() throws Exception {
|
||||
ImmutableList<BillingEvent> inputRows = getInputEvents();
|
||||
PCollection<BillingEvent> input = testPipeline.apply(Create.of(inputRows));
|
||||
invoicingPipeline.applyTerminalTransforms(input, StaticValueProvider.of("2017-10"));
|
||||
testPipeline.run();
|
||||
void testSuccess_makeQuery() {
|
||||
String query = InvoicingPipeline.makeQuery("2017-10", "my-project-id");
|
||||
assertThat(query)
|
||||
.isEqualTo(TestDataHelper.loadFile(this.getClass(), "billing_events_test.sql"));
|
||||
// This is necessary because the TestPipelineExtension verifies that the pipelien is run.
|
||||
pipeline.run();
|
||||
}
|
||||
|
||||
for (Entry<String, ImmutableList<String>> entry : getExpectedDetailReportMap().entrySet()) {
|
||||
@Test
|
||||
void testSuccess_saveInvoiceCsv() throws Exception {
|
||||
InvoicingPipeline.saveInvoiceCsv(billingEvents, options);
|
||||
pipeline.run().waitUntilFinish();
|
||||
ImmutableList<String> overallInvoice = resultFileContents("REG-INV-2017-10.csv");
|
||||
assertThat(overallInvoice.get(0))
|
||||
.isEqualTo(
|
||||
"StartDate,EndDate,ProductAccountKey,Amount,AmountCurrency,BillingProductCode,"
|
||||
+ "SalesChannel,LineItemType,UsageGroupingKey,Quantity,Description,UnitPrice,"
|
||||
+ "UnitPriceCurrency,PONumber");
|
||||
assertThat(overallInvoice.subList(1, overallInvoice.size()))
|
||||
.containsExactlyElementsIn(EXPECTED_INVOICE_OUTPUT);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_saveDetailedCsv() throws Exception {
|
||||
InvoicingPipeline.saveDetailedCsv(billingEvents, options);
|
||||
pipeline.run().waitUntilFinish();
|
||||
for (Entry<String, ImmutableList<String>> entry : EXPECTED_DETAILED_REPORT_MAP.entrySet()) {
|
||||
ImmutableList<String> detailReport = resultFileContents(entry.getKey());
|
||||
assertThat(detailReport.get(0))
|
||||
.isEqualTo(
|
||||
|
|
@ -206,22 +209,14 @@ class InvoicingPipelineTest {
|
|||
assertThat(detailReport.subList(1, detailReport.size()))
|
||||
.containsExactlyElementsIn(entry.getValue());
|
||||
}
|
||||
|
||||
ImmutableList<String> overallInvoice = resultFileContents("REG-INV-2017-10.csv");
|
||||
assertThat(overallInvoice.get(0))
|
||||
.isEqualTo(
|
||||
"StartDate,EndDate,ProductAccountKey,Amount,AmountCurrency,BillingProductCode,"
|
||||
+ "SalesChannel,LineItemType,UsageGroupingKey,Quantity,Description,UnitPrice,"
|
||||
+ "UnitPriceCurrency,PONumber");
|
||||
assertThat(overallInvoice.subList(1, overallInvoice.size()))
|
||||
.containsExactlyElementsIn(getExpectedInvoiceOutput());
|
||||
}
|
||||
|
||||
/** Returns the text contents of a file under the beamBucket/results directory. */
|
||||
private ImmutableList<String> resultFileContents(String filename) throws Exception {
|
||||
File resultFile =
|
||||
new File(
|
||||
String.format("%s/invoices/2017-10/%s", tmpDir.toAbsolutePath().toString(), filename));
|
||||
String.format(
|
||||
"%s/invoices/2017-10/%s", billingBucketUrl.getAbsolutePath().toString(), filename));
|
||||
return ImmutableList.copyOf(
|
||||
ResourceUtils.readResourceUtf8(resultFile.toURI().toURL()).split("\n"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,73 +0,0 @@
|
|||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam.invoicing;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import google.registry.testing.TestDataHelper;
|
||||
import org.apache.beam.sdk.io.DefaultFilenamePolicy.Params;
|
||||
import org.apache.beam.sdk.io.FileBasedSink;
|
||||
import org.apache.beam.sdk.options.ValueProvider;
|
||||
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
|
||||
import org.apache.beam.sdk.transforms.SerializableFunction;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Unit tests for {@link InvoicingUtils}. */
|
||||
class InvoicingUtilsTest {
|
||||
|
||||
@Test
|
||||
void testDestinationFunction_generatesProperFileParams() {
|
||||
SerializableFunction<BillingEvent, Params> destinationFunction =
|
||||
InvoicingUtils.makeDestinationFunction("my/directory", StaticValueProvider.of("2017-10"));
|
||||
|
||||
BillingEvent billingEvent = mock(BillingEvent.class);
|
||||
// We mock BillingEvent to make the test independent of the implementation of toFilename()
|
||||
when(billingEvent.toFilename(any())).thenReturn("invoice_details_2017-10_registrar_tld");
|
||||
|
||||
assertThat(destinationFunction.apply(billingEvent))
|
||||
.isEqualTo(
|
||||
new Params()
|
||||
.withShardTemplate("")
|
||||
.withSuffix(".csv")
|
||||
.withBaseFilename(
|
||||
FileBasedSink.convertToFileResourceIfPossible(
|
||||
"my/directory/2017-10/invoice_details_2017-10_registrar_tld")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testEmptyDestinationParams() {
|
||||
assertThat(InvoicingUtils.makeEmptyDestinationParams("my/directory"))
|
||||
.isEqualTo(
|
||||
new Params()
|
||||
.withBaseFilename(
|
||||
FileBasedSink.convertToFileResourceIfPossible("my/directory/FAILURES")));
|
||||
}
|
||||
|
||||
/** Asserts that the instantiated sql template matches a golden expected file. */
|
||||
@Test
|
||||
void testMakeQueryProvider() {
|
||||
ValueProvider<String> queryProvider =
|
||||
InvoicingUtils.makeQueryProvider(StaticValueProvider.of("2017-10"), "my-project-id");
|
||||
assertThat(queryProvider.get()).isEqualTo(loadFile("billing_events_test.sql"));
|
||||
}
|
||||
|
||||
/** Returns a {@link String} from a file in the {@code billing/testdata/} directory. */
|
||||
private static String loadFile(String filename) {
|
||||
return TestDataHelper.loadFile(InvoicingUtilsTest.class, filename);
|
||||
}
|
||||
}
|
||||
|
|
@ -17,94 +17,57 @@ package google.registry.reporting.billing;
|
|||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
||||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.dataflow.Dataflow;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Templates;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Templates.Launch;
|
||||
import com.google.api.services.dataflow.model.Job;
|
||||
import com.google.api.services.dataflow.model.LaunchTemplateParameters;
|
||||
import com.google.api.services.dataflow.model.LaunchTemplateResponse;
|
||||
import com.google.api.services.dataflow.model.RuntimeEnvironment;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.beam.BeamActionTestBase;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import java.io.IOException;
|
||||
import org.joda.time.YearMonth;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
|
||||
/** Unit tests for {@link google.registry.reporting.billing.GenerateInvoicesAction}. */
|
||||
class GenerateInvoicesActionTest {
|
||||
class GenerateInvoicesActionTest extends BeamActionTestBase {
|
||||
|
||||
@RegisterExtension
|
||||
final AppEngineExtension appEngine = AppEngineExtension.builder().withTaskQueue().build();
|
||||
|
||||
private Dataflow dataflow;
|
||||
private Projects projects;
|
||||
private Templates templates;
|
||||
private Launch launch;
|
||||
private FakeResponse response;
|
||||
private BillingEmailUtils emailUtils;
|
||||
private final BillingEmailUtils emailUtils = mock(BillingEmailUtils.class);
|
||||
private FakeClock clock = new FakeClock();
|
||||
private GenerateInvoicesAction action;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws IOException {
|
||||
dataflow = mock(Dataflow.class);
|
||||
projects = mock(Projects.class);
|
||||
templates = mock(Templates.class);
|
||||
launch = mock(Launch.class);
|
||||
emailUtils = mock(BillingEmailUtils.class);
|
||||
when(dataflow.projects()).thenReturn(projects);
|
||||
when(projects.templates()).thenReturn(templates);
|
||||
when(templates.launch(any(String.class), any(LaunchTemplateParameters.class)))
|
||||
.thenReturn(launch);
|
||||
when(launch.setGcsPath(any(String.class))).thenReturn(launch);
|
||||
|
||||
response = new FakeResponse();
|
||||
Job job = new Job();
|
||||
job.setId("12345");
|
||||
when(launch.execute()).thenReturn(new LaunchTemplateResponse().setJob(job));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLaunchTemplateJob_withPublish() throws Exception {
|
||||
action =
|
||||
new GenerateInvoicesAction(
|
||||
"test-project",
|
||||
"gs://test-project-beam",
|
||||
"gs://test-project-beam/templates/invoicing",
|
||||
"us-east1-c",
|
||||
"test-region",
|
||||
"staging_bucket",
|
||||
"billing_bucket",
|
||||
"REG-INV",
|
||||
true,
|
||||
new YearMonth(2017, 10),
|
||||
dataflow,
|
||||
emailUtils,
|
||||
clock,
|
||||
response,
|
||||
emailUtils);
|
||||
dataflow);
|
||||
action.run();
|
||||
LaunchTemplateParameters expectedParams =
|
||||
new LaunchTemplateParameters()
|
||||
.setJobName("invoicing-2017-10")
|
||||
.setEnvironment(
|
||||
new RuntimeEnvironment()
|
||||
.setZone("us-east1-c")
|
||||
.setTempLocation("gs://test-project-beam/temporary"))
|
||||
.setParameters(ImmutableMap.of("yearMonth", "2017-10"));
|
||||
verify(templates).launch("test-project", expectedParams);
|
||||
verify(launch).setGcsPath("gs://test-project-beam/templates/invoicing");
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
assertThat(response.getPayload()).isEqualTo("Launched dataflow template.");
|
||||
|
||||
TaskMatcher matcher =
|
||||
new TaskMatcher()
|
||||
.url("/_dr/task/publishInvoices")
|
||||
.method("POST")
|
||||
.param("jobId", "12345")
|
||||
.param("jobId", "jobid")
|
||||
.param("yearMonth", "2017-10");
|
||||
assertTasksEnqueued("beam-reporting", matcher);
|
||||
}
|
||||
|
|
@ -114,47 +77,43 @@ class GenerateInvoicesActionTest {
|
|||
action =
|
||||
new GenerateInvoicesAction(
|
||||
"test-project",
|
||||
"gs://test-project-beam",
|
||||
"gs://test-project-beam/templates/invoicing",
|
||||
"us-east1-c",
|
||||
"test-region",
|
||||
"staging_bucket",
|
||||
"billing_bucket",
|
||||
"REG-INV",
|
||||
false,
|
||||
new YearMonth(2017, 10),
|
||||
dataflow,
|
||||
emailUtils,
|
||||
clock,
|
||||
response,
|
||||
emailUtils);
|
||||
dataflow);
|
||||
action.run();
|
||||
LaunchTemplateParameters expectedParams =
|
||||
new LaunchTemplateParameters()
|
||||
.setJobName("invoicing-2017-10")
|
||||
.setEnvironment(
|
||||
new RuntimeEnvironment()
|
||||
.setZone("us-east1-c")
|
||||
.setTempLocation("gs://test-project-beam/temporary"))
|
||||
.setParameters(ImmutableMap.of("yearMonth", "2017-10"));
|
||||
verify(templates).launch("test-project", expectedParams);
|
||||
verify(launch).setGcsPath("gs://test-project-beam/templates/invoicing");
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
|
||||
assertThat(response.getStatus()).isEqualTo(SC_OK);
|
||||
assertThat(response.getPayload()).isEqualTo("Launched dataflow template.");
|
||||
assertNoTasksEnqueued();
|
||||
assertNoTasksEnqueued("beam-reporting");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCaughtIOException() throws IOException {
|
||||
when(launch.execute()).thenThrow(new IOException("expected"));
|
||||
when(launch.execute()).thenThrow(new IOException("Pipeline error"));
|
||||
action =
|
||||
new GenerateInvoicesAction(
|
||||
"test-project",
|
||||
"gs://test-project-beam",
|
||||
"gs://test-project-beam/templates/invoicing",
|
||||
"us-east1-c",
|
||||
true,
|
||||
"test-region",
|
||||
"staging_bucket",
|
||||
"billing_bucket",
|
||||
"REG-INV",
|
||||
false,
|
||||
new YearMonth(2017, 10),
|
||||
dataflow,
|
||||
emailUtils,
|
||||
clock,
|
||||
response,
|
||||
emailUtils);
|
||||
dataflow);
|
||||
action.run();
|
||||
assertThat(response.getStatus()).isEqualTo(500);
|
||||
assertThat(response.getPayload()).isEqualTo("Template launch failed: expected");
|
||||
verify(emailUtils).sendAlertEmail("Template Launch failed due to expected");
|
||||
assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
|
||||
assertThat(response.getPayload()).isEqualTo("Template launch failed: Pipeline error");
|
||||
verify(emailUtils).sendAlertEmail("Template Launch failed due to Pipeline error");
|
||||
assertNoTasksEnqueued("beam-reporting");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,8 +26,9 @@ import static org.mockito.Mockito.when;
|
|||
|
||||
import com.google.api.services.dataflow.Dataflow;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Jobs;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Jobs.Get;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations.Jobs;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations.Jobs.Get;
|
||||
import com.google.api.services.dataflow.model.Job;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
|
|
@ -42,11 +43,14 @@ import org.junit.jupiter.api.extension.RegisterExtension;
|
|||
/** Unit tests for {@link PublishInvoicesAction}. */
|
||||
class PublishInvoicesActionTest {
|
||||
|
||||
private Get get;
|
||||
private BillingEmailUtils emailUtils;
|
||||
|
||||
private Job expectedJob;
|
||||
private FakeResponse response;
|
||||
private final Dataflow dataflow = mock(Dataflow.class);
|
||||
private final Projects projects = mock(Projects.class);
|
||||
private final Locations locations = mock(Locations.class);
|
||||
private final Jobs jobs = mock(Jobs.class);
|
||||
private final Get get = mock(Get.class);
|
||||
private final Job expectedJob = new Job();
|
||||
private final BillingEmailUtils emailUtils = mock(BillingEmailUtils.class);
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private PublishInvoicesAction uploadAction;
|
||||
|
||||
@RegisterExtension
|
||||
|
|
@ -54,20 +58,21 @@ class PublishInvoicesActionTest {
|
|||
|
||||
@BeforeEach
|
||||
void beforeEach() throws IOException {
|
||||
Dataflow dataflow = mock(Dataflow.class);
|
||||
Projects projects = mock(Projects.class);
|
||||
Jobs jobs = mock(Jobs.class);
|
||||
get = mock(Get.class);
|
||||
when(dataflow.projects()).thenReturn(projects);
|
||||
when(projects.jobs()).thenReturn(jobs);
|
||||
when(jobs.get("test-project", "12345")).thenReturn(get);
|
||||
expectedJob = new Job();
|
||||
when(projects.locations()).thenReturn(locations);
|
||||
when(locations.jobs()).thenReturn(jobs);
|
||||
when(jobs.get("test-project", "test-region", "12345")).thenReturn(get);
|
||||
when(get.execute()).thenReturn(expectedJob);
|
||||
emailUtils = mock(BillingEmailUtils.class);
|
||||
response = new FakeResponse();
|
||||
|
||||
uploadAction =
|
||||
new PublishInvoicesAction(
|
||||
"test-project", "12345", emailUtils, dataflow, response, new YearMonth(2017, 10));
|
||||
"test-project",
|
||||
"test-region",
|
||||
"12345",
|
||||
emailUtils,
|
||||
dataflow,
|
||||
response,
|
||||
new YearMonth(2017, 10));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
|
|||
|
|
@ -19,61 +19,27 @@ import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
|
|||
import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static org.apache.http.HttpStatus.SC_OK;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.services.dataflow.Dataflow;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations.FlexTemplates;
|
||||
import com.google.api.services.dataflow.Dataflow.Projects.Locations.FlexTemplates.Launch;
|
||||
import com.google.api.services.dataflow.model.Job;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
|
||||
import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
|
||||
import com.google.common.net.MediaType;
|
||||
import google.registry.beam.BeamActionTestBase;
|
||||
import google.registry.testing.AppEngineExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.TaskQueueHelper.TaskMatcher;
|
||||
import java.io.IOException;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
|
||||
/** Unit tests for {@link GenerateSpec11ReportAction}. */
|
||||
@MockitoSettings(strictness = Strictness.STRICT_STUBS)
|
||||
class GenerateSpec11ReportActionTest {
|
||||
class GenerateSpec11ReportActionTest extends BeamActionTestBase {
|
||||
|
||||
@RegisterExtension
|
||||
final AppEngineExtension appEngine = AppEngineExtension.builder().withTaskQueue().build();
|
||||
|
||||
private FakeResponse response = new FakeResponse();
|
||||
private Dataflow dataflow = mock(Dataflow.class);
|
||||
private Projects projects = mock(Projects.class);
|
||||
private Locations locations = mock(Locations.class);
|
||||
private FlexTemplates templates = mock(FlexTemplates.class);
|
||||
private Launch launch = mock(Launch.class);
|
||||
private LaunchFlexTemplateResponse launchResponse =
|
||||
new LaunchFlexTemplateResponse().setJob(new Job().setId("jobid"));
|
||||
|
||||
private final FakeClock clock = new FakeClock(DateTime.parse("2018-06-11T12:23:56Z"));
|
||||
private GenerateSpec11ReportAction action;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() throws IOException {
|
||||
when(dataflow.projects()).thenReturn(projects);
|
||||
when(projects.locations()).thenReturn(locations);
|
||||
when(locations.flexTemplates()).thenReturn(templates);
|
||||
when(templates.launch(anyString(), anyString(), any(LaunchFlexTemplateRequest.class)))
|
||||
.thenReturn(launch);
|
||||
when(launch.execute()).thenReturn(launchResponse);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFailure_dataflowFailure() throws IOException {
|
||||
action =
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue