Clarify that we are actually using the client ID in Spec11

The Registrar object's @ID is on its clientId field, so that's what we're actually using here, not the name.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=241936005
This commit is contained in:
gbrodman 2019-04-04 08:36:33 -07:00 committed by jianglai
parent 33cdc3ddda
commit d217939894
4 changed files with 20 additions and 19 deletions

View file

@ -73,7 +73,7 @@ public class Spec11Pipeline implements Serializable {
/** The JSON object field into which we put the registrar's e-mail address for Spec11 reports. */ /** The JSON object field into which we put the registrar's e-mail address for Spec11 reports. */
public static final String REGISTRAR_EMAIL_FIELD = "registrarEmailAddress"; public static final String REGISTRAR_EMAIL_FIELD = "registrarEmailAddress";
/** The JSON object field into which we put the registrar's name for Spec11 reports. */ /** The JSON object field into which we put the registrar's name for Spec11 reports. */
public static final String REGISTRAR_NAME_FIELD = "registrarName"; public static final String REGISTRAR_CLIENT_ID_FIELD = "registrarClientId";
/** The JSON object field we put the threat match array for Spec11 reports. */ /** The JSON object field we put the threat match array for Spec11 reports. */
public static final String THREAT_MATCHES_FIELD = "threatMatches"; public static final String THREAT_MATCHES_FIELD = "threatMatches";
@ -172,31 +172,31 @@ public class Spec11Pipeline implements Serializable {
domains.apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn)); domains.apply("Run through SafeBrowsingAPI", ParDo.of(evaluateSafeBrowsingFn));
subdomains subdomains
.apply( .apply(
"Map registrar name to email/ThreatMatch pair", "Map registrar client ID to email/ThreatMatch pair",
MapElements.into( MapElements.into(
TypeDescriptors.kvs( TypeDescriptors.kvs(
TypeDescriptors.strings(), TypeDescriptor.of(EmailAndThreatMatch.class))) TypeDescriptors.strings(), TypeDescriptor.of(EmailAndThreatMatch.class)))
.via( .via(
(KV<Subdomain, ThreatMatch> kv) -> (KV<Subdomain, ThreatMatch> kv) ->
KV.of( KV.of(
kv.getKey().registrarName(), kv.getKey().registrarClientId(),
EmailAndThreatMatch.create( EmailAndThreatMatch.create(
kv.getKey().registrarEmailAddress(), kv.getValue())))) kv.getKey().registrarEmailAddress(), kv.getValue()))))
.apply("Group by registrar name", GroupByKey.create()) .apply("Group by registrar client ID", GroupByKey.create())
.apply( .apply(
"Convert results to JSON format", "Convert results to JSON format",
MapElements.into(TypeDescriptors.strings()) MapElements.into(TypeDescriptors.strings())
.via( .via(
(KV<String, Iterable<EmailAndThreatMatch>> kv) -> { (KV<String, Iterable<EmailAndThreatMatch>> kv) -> {
String registrarName = kv.getKey(); String clientId = kv.getKey();
checkArgument( checkArgument(
kv.getValue().iterator().hasNext(), kv.getValue().iterator().hasNext(),
String.format( String.format(
"Registrar named %s had no corresponding threats", registrarName)); "Registrar with ID %s had no corresponding threats", clientId));
String email = kv.getValue().iterator().next().email(); String email = kv.getValue().iterator().next().email();
JSONObject output = new JSONObject(); JSONObject output = new JSONObject();
try { try {
output.put(REGISTRAR_NAME_FIELD, registrarName); output.put(REGISTRAR_CLIENT_ID_FIELD, clientId);
output.put(REGISTRAR_EMAIL_FIELD, email); output.put(REGISTRAR_EMAIL_FIELD, email);
JSONArray threatMatchArray = new JSONArray(); JSONArray threatMatchArray = new JSONArray();
for (EmailAndThreatMatch emailAndThreatMatch : kv.getValue()) { for (EmailAndThreatMatch emailAndThreatMatch : kv.getValue()) {

View file

@ -36,12 +36,12 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
public abstract class Subdomain implements Serializable { public abstract class Subdomain implements Serializable {
private static final ImmutableList<String> FIELD_NAMES = private static final ImmutableList<String> FIELD_NAMES =
ImmutableList.of("fullyQualifiedDomainName", "registrarName", "registrarEmailAddress"); ImmutableList.of("fullyQualifiedDomainName", "registrarClientId", "registrarEmailAddress");
/** Returns the fully qualified domain name. */ /** Returns the fully qualified domain name. */
abstract String fullyQualifiedDomainName(); abstract String fullyQualifiedDomainName();
/** Returns the name of the associated registrar for this domain. */ /** Returns the client ID of the associated registrar for this domain. */
abstract String registrarName(); abstract String registrarClientId();
/** Returns the email address of the registrar associated with this domain. */ /** Returns the email address of the registrar associated with this domain. */
abstract String registrarEmailAddress(); abstract String registrarEmailAddress();
@ -57,7 +57,7 @@ public abstract class Subdomain implements Serializable {
GenericRecord record = schemaAndRecord.getRecord(); GenericRecord record = schemaAndRecord.getRecord();
return create( return create(
extractField(record, "fullyQualifiedDomainName"), extractField(record, "fullyQualifiedDomainName"),
extractField(record, "registrarName"), extractField(record, "registrarClientId"),
extractField(record, "registrarEmailAddress")); extractField(record, "registrarEmailAddress"));
} }
@ -69,8 +69,9 @@ public abstract class Subdomain implements Serializable {
*/ */
@VisibleForTesting @VisibleForTesting
static Subdomain create( static Subdomain create(
String fullyQualifiedDomainName, String registrarName, String registrarEmailAddress) { String fullyQualifiedDomainName, String registrarClientId, String registrarEmailAddress) {
return new AutoValue_Subdomain(fullyQualifiedDomainName, registrarName, registrarEmailAddress); return new AutoValue_Subdomain(
fullyQualifiedDomainName, registrarClientId, registrarEmailAddress);
} }
} }

View file

@ -20,7 +20,7 @@
SELECT SELECT
domain.fullyQualifiedDomainName AS fullyQualifiedDomainName, domain.fullyQualifiedDomainName AS fullyQualifiedDomainName,
registrar.name AS registrarName, registrar.clientId AS registrarClientId,
COALESCE(registrar.emailAddress, '') AS registrarEmailAddress COALESCE(registrar.emailAddress, '') AS registrarEmailAddress
FROM ( ( FROM ( (
SELECT SELECT
@ -37,13 +37,13 @@ FROM ( (
OR deletionTime > CURRENT_TIMESTAMP)) AS domain OR deletionTime > CURRENT_TIMESTAMP)) AS domain
JOIN ( JOIN (
SELECT SELECT
__key__.name AS name, __key__.name AS clientId,
emailAddress emailAddress
FROM FROM
`%PROJECT_ID%.%DATASTORE_EXPORT_DATASET%.%REGISTRAR_TABLE%` `%PROJECT_ID%.%DATASTORE_EXPORT_DATASET%.%REGISTRAR_TABLE%`
WHERE WHERE
type = 'REAL') AS registrar type = 'REAL') AS registrar
ON ON
domain.currentSponsorClientId = registrar.name) domain.currentSponsorClientId = registrar.clientId)
ORDER BY ORDER BY
creationTime DESC creationTime DESC

View file

@ -151,7 +151,7 @@ public class Spec11PipelineTest {
JSONObject noEmailRegistrarJSON = new JSONObject(sortedLines.get(0)); JSONObject noEmailRegistrarJSON = new JSONObject(sortedLines.get(0));
assertThat(noEmailRegistrarJSON.get("registrarEmailAddress")).isEqualTo(""); assertThat(noEmailRegistrarJSON.get("registrarEmailAddress")).isEqualTo("");
assertThat(noEmailRegistrarJSON.get("registrarName")).isEqualTo("noEmailRegistrar"); assertThat(noEmailRegistrarJSON.get("registrarClientId")).isEqualTo("noEmailRegistrar");
assertThat(noEmailRegistrarJSON.has("threatMatches")).isTrue(); assertThat(noEmailRegistrarJSON.has("threatMatches")).isTrue();
JSONArray noEmailThreatMatch = noEmailRegistrarJSON.getJSONArray("threatMatches"); JSONArray noEmailThreatMatch = noEmailRegistrarJSON.getJSONArray("threatMatches");
assertThat(noEmailThreatMatch.length()).isEqualTo(1); assertThat(noEmailThreatMatch.length()).isEqualTo(1);
@ -162,7 +162,7 @@ public class Spec11PipelineTest {
JSONObject someRegistrarJSON = new JSONObject(sortedLines.get(1)); JSONObject someRegistrarJSON = new JSONObject(sortedLines.get(1));
assertThat(someRegistrarJSON.get("registrarEmailAddress")).isEqualTo("fake@someRegistrar.com"); assertThat(someRegistrarJSON.get("registrarEmailAddress")).isEqualTo("fake@someRegistrar.com");
assertThat(someRegistrarJSON.get("registrarName")).isEqualTo("someRegistrar"); assertThat(someRegistrarJSON.get("registrarClientId")).isEqualTo("someRegistrar");
assertThat(someRegistrarJSON.has("threatMatches")).isTrue(); assertThat(someRegistrarJSON.has("threatMatches")).isTrue();
JSONArray someThreatMatch = someRegistrarJSON.getJSONArray("threatMatches"); JSONArray someThreatMatch = someRegistrarJSON.getJSONArray("threatMatches");
assertThat(someThreatMatch.length()).isEqualTo(1); assertThat(someThreatMatch.length()).isEqualTo(1);
@ -174,7 +174,7 @@ public class Spec11PipelineTest {
// theRegistrar has two ThreatMatches, we have to parse it explicitly // theRegistrar has two ThreatMatches, we have to parse it explicitly
JSONObject theRegistrarJSON = new JSONObject(sortedLines.get(2)); JSONObject theRegistrarJSON = new JSONObject(sortedLines.get(2));
assertThat(theRegistrarJSON.get("registrarEmailAddress")).isEqualTo("fake@theRegistrar.com"); assertThat(theRegistrarJSON.get("registrarEmailAddress")).isEqualTo("fake@theRegistrar.com");
assertThat(theRegistrarJSON.get("registrarName")).isEqualTo("theRegistrar"); assertThat(theRegistrarJSON.get("registrarClientId")).isEqualTo("theRegistrar");
assertThat(theRegistrarJSON.has("threatMatches")).isTrue(); assertThat(theRegistrarJSON.has("threatMatches")).isTrue();
JSONArray theThreatMatches = theRegistrarJSON.getJSONArray("threatMatches"); JSONArray theThreatMatches = theRegistrarJSON.getJSONArray("threatMatches");
assertThat(theThreatMatches.length()).isEqualTo(2); assertThat(theThreatMatches.length()).isEqualTo(2);