mirror of
https://github.com/google/nomulus.git
synced 2025-05-18 18:29:36 +02:00
Add preliminary spec11 monthly pipeline
This adds the scaffolding for a basic Spec11 pipeline- it gathers all domains from all time for a given project and counts how many there are. I've factored out a few common utilities for beam pipelines to avoid excessive duplication. Future CLs will: - Actually process domains via the SafeBrowsing API - Generate a real spec11 report - Template queries based on the input YearMonth - Abstract more commonalities across beam pipelines to reduce boilerplate when adding new pipelines. TESTED: FOSS test passed, and ran successfully on alpha ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=205997741
This commit is contained in:
parent
ded40851d3
commit
d199b383e5
14 changed files with 252 additions and 38 deletions
86
javatests/google/registry/beam/BeamUtilsTest.java
Normal file
86
javatests/google/registry/beam/BeamUtilsTest.java
Normal file
|
@ -0,0 +1,86 @@
|
|||
// Copyright 2018 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.beam;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.JUnitBackports.assertThrows;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.apache.avro.Schema;
|
||||
import org.apache.avro.generic.GenericData;
|
||||
import org.apache.avro.generic.GenericRecord;
|
||||
import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
||||
/** Unit tests for {@link BeamUtils} */
|
||||
@RunWith(JUnit4.class)
|
||||
public class BeamUtilsTest {
|
||||
|
||||
private static final String GENERIC_SCHEMA =
|
||||
"{\"name\": \"AnObject\", "
|
||||
+ "\"type\": \"record\", "
|
||||
+ "\"fields\": ["
|
||||
+ "{\"name\": \"aString\", \"type\": \"string\"},"
|
||||
+ "{\"name\": \"aFloat\", \"type\": \"float\"}"
|
||||
+ "]}";
|
||||
|
||||
private SchemaAndRecord schemaAndRecord;
|
||||
|
||||
@Before
|
||||
public void initializeRecord() {
|
||||
// Create a record with a given JSON schema.
|
||||
GenericRecord record = new GenericData.Record(new Schema.Parser().parse(GENERIC_SCHEMA));
|
||||
record.put("aString", "hello world");
|
||||
record.put("aFloat", 2.54);
|
||||
schemaAndRecord = new SchemaAndRecord(record, null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtractField_fieldExists_returnsExpectedStringValues() {
|
||||
assertThat(BeamUtils.extractField(schemaAndRecord.getRecord(), "aString"))
|
||||
.isEqualTo("hello world");
|
||||
assertThat(BeamUtils.extractField(schemaAndRecord.getRecord(), "aFloat")).isEqualTo("2.54");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExtractField_fieldDoesntExist_returnsNull() {
|
||||
schemaAndRecord.getRecord().put("aFloat", null);
|
||||
assertThat(BeamUtils.extractField(schemaAndRecord.getRecord(), "aFloat")).isEqualTo("null");
|
||||
assertThat(BeamUtils.extractField(schemaAndRecord.getRecord(), "missing")).isEqualTo("null");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckFieldsNotNull_noExceptionIfAllPresent() {
|
||||
BeamUtils.checkFieldsNotNull(ImmutableList.of("aString", "aFloat"), schemaAndRecord);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckFieldsNotNull_fieldMissing_throwsException() {
|
||||
IllegalStateException expected =
|
||||
assertThrows(
|
||||
IllegalStateException.class,
|
||||
() ->
|
||||
BeamUtils.checkFieldsNotNull(
|
||||
ImmutableList.of("aString", "aFloat", "notAField"), schemaAndRecord));
|
||||
assertThat(expected)
|
||||
.hasMessageThat()
|
||||
.isEqualTo(
|
||||
"Read unexpected null value for field(s) notAField for record "
|
||||
+ "{\"aString\": \"hello world\", \"aFloat\": 2.54}");
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue