Add @ReportedOn annotation for BigQuery exports

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=142038227

java/google/registry/export/ExportConstants.java

@@ -18,66 +18,17 @@ import static com.google.common.base.Predicates.not;
 import static google.registry.model.EntityClasses.CLASS_TO_KIND_FUNCTION;
 import static google.registry.util.TypeUtils.hasAnnotation;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Ordering;
 import google.registry.model.EntityClasses;
-import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.annotations.VirtualEntity;
-import google.registry.model.billing.BillingEvent.Cancellation;
-import google.registry.model.billing.BillingEvent.Modification;
-import google.registry.model.billing.BillingEvent.OneTime;
-import google.registry.model.billing.BillingEvent.Recurring;
-import google.registry.model.billing.RegistrarCredit;
-import google.registry.model.billing.RegistrarCreditBalance;
-import google.registry.model.contact.ContactResource;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.LrpTokenEntity;
-import google.registry.model.host.HostResource;
-import google.registry.model.index.DomainApplicationIndex;
-import google.registry.model.index.EppResourceIndex;
-import google.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex;
-import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
-import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
-import google.registry.model.registrar.Registrar;
-import google.registry.model.registrar.RegistrarContact;
-import google.registry.model.registry.Registry;
-import google.registry.model.registry.label.PremiumList;
-import google.registry.model.registry.label.PremiumList.PremiumListEntry;
-import google.registry.model.reporting.HistoryEntry;
 
 /** Constants related to export code. */
 public final class ExportConstants {
 
-  /** Set of entity classes to export into BigQuery for reporting purposes. */
-  @VisibleForTesting
-  @SuppressWarnings("unchecked") // varargs
-  static final ImmutableSet<Class<? extends ImmutableObject>> REPORTING_ENTITY_CLASSES =
-      ImmutableSet.of(
-          Cancellation.class,
-          ContactResource.class,
-          DomainApplicationIndex.class,
-          DomainBase.class,
-          EppResourceIndex.class,
-          ForeignKeyContactIndex.class,
-          ForeignKeyDomainIndex.class,
-          ForeignKeyHostIndex.class,
-          HistoryEntry.class,
-          HostResource.class,
-          LrpTokenEntity.class,
-          Modification.class,
-          OneTime.class,
-          PremiumList.class,
-          PremiumListEntry.class,
-          Recurring.class,
-          Registrar.class,
-          RegistrarContact.class,
-          RegistrarCredit.class,
-          RegistrarCreditBalance.class,
-          Registry.class);
-
   /** Returns the names of kinds to include in datastore backups. */
   public static ImmutableSet<String> getBackupKinds() {
     // Back up all entity classes that aren't annotated with @VirtualEntity (never even persisted
@@ -91,7 +42,9 @@ public final class ExportConstants {
 
   /** Returns the names of kinds to import into reporting tools (e.g. BigQuery). */
   public static ImmutableSet<String> getReportingKinds() {
-    return FluentIterable.from(REPORTING_ENTITY_CLASSES)
+    return FluentIterable.from(EntityClasses.ALL_CLASSES)
+        .filter(hasAnnotation(ReportedOn.class))
+        .filter(not(hasAnnotation(VirtualEntity.class)))
         .transform(CLASS_TO_KIND_FUNCTION)
         .toSortedSet(Ordering.natural());
   }

java/google/registry/model/annotations/ReportedOn.java

@@ -0,0 +1,28 @@
+// Copyright 2016 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.annotations;
+
+import com.googlecode.objectify.annotation.Entity;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation for an Objectify {@link Entity} to indicate that it should be exported to BigQuery.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface ReportedOn {}

java/google/registry/model/billing/BillingEvent.java

@@ -37,6 +37,7 @@ import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.TimeOfYear;
 import google.registry.model.domain.GracePeriod;
 import google.registry.model.domain.rgp.GracePeriodStatus;
@@ -198,6 +199,7 @@ public abstract class BillingEvent extends ImmutableObject
   }
 
   /** A one-time billable event. */
+  @ReportedOn
   @Entity
   public static class OneTime extends BillingEvent {
 
@@ -328,6 +330,7 @@ public abstract class BillingEvent extends ImmutableObject
    * recurring event might change and each time we bill for it we need to bill at the current cost,
    * not the value that was in use at the time the recurrence was created.
    */
+  @ReportedOn
   @Entity
   public static class Recurring extends BillingEvent {
 
@@ -400,6 +403,7 @@ public abstract class BillingEvent extends ImmutableObject
    * <p>This is implemented as a separate event rather than a bit on BillingEvent in order to
    * preserve the immutability of billing events.
    */
+  @ReportedOn
   @Entity
   public static class Cancellation extends BillingEvent {
 
@@ -510,6 +514,7 @@ public abstract class BillingEvent extends ImmutableObject
   /**
    * An event representing a modification of an existing one-time billing event.
    */
+  @ReportedOn
   @Entity
   public static class Modification extends BillingEvent {
 

java/google/registry/model/billing/RegistrarCredit.java

@@ -31,12 +31,14 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registry.Registry;
 import org.joda.money.CurrencyUnit;
 import org.joda.time.DateTime;
 
 /** A per-registrar billing credit, applied toward future charges for registrar activity. */
+@ReportedOn
 @Entity
 public final class RegistrarCredit extends ImmutableObject implements Buildable {
 

java/google/registry/model/billing/RegistrarCreditBalance.java

@@ -32,6 +32,7 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.ReportedOn;
 import java.util.HashMap;
 import java.util.Map;
 import org.joda.money.CurrencyUnit;
@@ -49,6 +50,7 @@ import org.joda.time.DateTime;
  * taking the balance object with the latest effective time that is before (before or at) T, and
  * breaking any ties by choosing the mostly recently written among those balances.
  */
+@ReportedOn
 @Entity
 public final class RegistrarCreditBalance extends ImmutableObject implements Buildable {
 

java/google/registry/model/contact/ContactResource.java

@@ -31,6 +31,7 @@ import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.EppResource.ResourceWithTransferData;
 import google.registry.model.annotations.ExternalMessagingName;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.PostalInfo.Type;
 import google.registry.model.transfer.TransferData;
 import java.util.List;
@@ -65,6 +66,7 @@ import org.joda.time.DateTime;
     "authInfo",
     "disclose" })
 @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+@ReportedOn
 @Entity
 @ExternalMessagingName("contact")
 public class ContactResource extends EppResource

java/google/registry/model/domain/DomainBase.java

@@ -42,6 +42,7 @@ import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DesignatedContact.Type;
 import google.registry.model.domain.launch.LaunchNotice;
@@ -54,6 +55,7 @@ import javax.xml.bind.annotation.XmlTransient;
 
 /** Shared base class for {@link DomainResource} and {@link DomainApplication}. */
 @XmlTransient
+@ReportedOn
 @Entity
 public abstract class DomainBase extends EppResource {
 

java/google/registry/model/domain/LrpTokenEntity.java

@@ -24,11 +24,13 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.Buildable;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.reporting.HistoryEntry;
 import java.util.Map;
 import java.util.Set;
 
 /** An entity representing a token distributed to eligible LRP registrants. */
+@ReportedOn
 @Entity
 public class LrpTokenEntity extends BackupGroupRoot implements Buildable {
 

java/google/registry/model/host/HostResource.java

@@ -32,6 +32,7 @@ import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
 import google.registry.model.annotations.ExternalMessagingName;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.domain.DomainResource;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.transfer.TransferData;
@@ -65,6 +66,7 @@ import org.joda.time.DateTime;
     "lastEppUpdateTime",
     "lastTransferTime" })
 @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+@ReportedOn
 @Entity
 @ExternalMessagingName("host")
 public class HostResource extends EppResource implements ForeignKeyedEppResource {

java/google/registry/model/index/DomainApplicationIndex.java

@@ -27,6 +27,7 @@ import com.googlecode.objectify.annotation.Cache;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.BackupGroupRoot;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.domain.DomainApplication;
 import google.registry.util.CollectionUtils;
 import java.util.Set;
@@ -38,6 +39,7 @@ import org.joda.time.DateTime;
  * resource is always kept up to date as additional domain applications are created, it is never
  * necessary to query them explicitly from Datastore.
  */
+@ReportedOn
 @Entity
 @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
 public class DomainApplicationIndex extends BackupGroupRoot {

java/google/registry/model/index/EppResourceIndex.java

@@ -24,8 +24,10 @@ import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.ReportedOn;
 
 /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */
+@ReportedOn
 @Entity
 public class EppResourceIndex extends BackupGroupRoot {
 

java/google/registry/model/index/ForeignKeyIndex.java

@@ -29,6 +29,7 @@ import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.EppResource;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainResource;
 import google.registry.model.host.HostResource;
@@ -45,16 +46,19 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
 
   /** The {@link ForeignKeyIndex} type for {@link ContactResource} entities. */
   @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+  @ReportedOn
   @Entity
   public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource> {}
 
   /** The {@link ForeignKeyIndex} type for {@link DomainResource} entities. */
   @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+  @ReportedOn
   @Entity
   public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainResource> {}
 
   /** The {@link ForeignKeyIndex} type for {@link HostResource} entities. */
   @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+  @ReportedOn
   @Entity
   public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource> {}
 

java/google/registry/model/registrar/Registrar.java

@@ -59,6 +59,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
 import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.util.CidrAddressBlock;
 import google.registry.util.NonFinalForTesting;
@@ -76,6 +77,7 @@ import org.joda.time.DateTime;
 
 /** Information about a registrar. */
 @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+@ReportedOn
 @Entity
 public class Registrar extends ImmutableObject implements Buildable, Jsonifiable {
 

java/google/registry/model/registrar/RegistrarContact.java

@@ -39,6 +39,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.annotations.ReportedOn;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Set;
@@ -52,6 +53,7 @@ import java.util.Set;
  * set to true.
  */
 @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+@ReportedOn
 @Entity
 public class RegistrarContact extends ImmutableObject implements Jsonifiable {
 

java/google/registry/model/registry/Registry.java

@@ -53,6 +53,7 @@ import google.registry.config.RegistryEnvironment;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
@@ -71,6 +72,7 @@ import org.joda.time.Interval;
 
 /** Persisted per-TLD configuration data. */
 @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
+@ReportedOn
 @Entity
 public class Registry extends ImmutableObject implements Buildable {
 

java/google/registry/model/registry/label/PremiumList.java

@@ -51,6 +51,7 @@ import com.googlecode.objectify.cmd.Query;
 import google.registry.config.RegistryEnvironment;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.annotations.VirtualEntity;
 import google.registry.model.registry.Registry;
 import java.util.List;
@@ -64,6 +65,7 @@ import org.joda.time.DateTime;
 /**
  * A premium list entity, persisted to Datastore, that is used to check domain label prices.
  */
+@ReportedOn
 @Entity
 @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
 public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.PremiumListEntry> {
@@ -192,6 +194,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
    * A premium list entry entity, persisted to Datastore.  Each instance represents the price of a
    * single label on a given TLD.
    */
+  @ReportedOn
   @Entity
   @Cache(expirationSeconds = RECOMMENDED_MEMCACHE_EXPIRATION)
   public static class PremiumListEntry extends DomainLabelEntry<Money, PremiumListEntry>

java/google/registry/model/reporting/HistoryEntry.java

@@ -24,11 +24,13 @@ import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.domain.Period;
 import google.registry.model.eppcommon.Trid;
 import org.joda.time.DateTime;
 
 /** A record of an EPP command that mutated a resource. */
+@ReportedOn
 @Entity
 public class HistoryEntry extends ImmutableObject implements Buildable {
 

javatests/google/registry/export/BUILD

@@ -12,6 +12,7 @@ java_library(
     srcs = glob(["*.java"]),
     resources = glob([
         "backup_kinds.txt",
+        "reporting_kinds.txt",
     ]),
     deps = [
         "//apiserving/discoverydata/bigquery:bigqueryv2",

javatests/google/registry/export/ExportConstantsTest.java

@@ -17,17 +17,15 @@ package google.registry.export;
 import static com.google.common.io.Resources.getResource;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
-import static java.nio.charset.StandardCharsets.UTF_8;
+import static google.registry.util.ResourceUtils.readResourceUtf8;
 
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.io.Resources;
 import com.google.re2j.Pattern;
-import com.googlecode.objectify.annotation.Entity;
-import google.registry.model.ImmutableObject;
 import java.net.URL;
 import java.util.List;
 import javax.annotation.Nullable;
@@ -41,10 +39,12 @@ public class ExportConstantsTest {
 
   private static final String GOLDEN_BACKUP_KINDS_FILENAME = "backup_kinds.txt";
 
+  private static final String GOLDEN_REPORTING_KINDS_FILENAME = "reporting_kinds.txt";
+
   private static final String UPDATE_INSTRUCTIONS_TEMPLATE = Joiner.on('\n').join(
       "",
       "---------------------------------------------------------------------------------",
-      "Your changes affect the list of backed-up kinds in the golden file:",
+      "Your changes affect the list of %s kinds in the golden file:",
       "  %s",
       "If these changes are desired, update the golden file with the following contents:",
       "=================================================================================",
@@ -56,38 +56,67 @@ public class ExportConstantsTest {
   public void testBackupKinds_matchGoldenBackupKindsFile() throws Exception {
     URL goldenBackupKindsResource =
         getResource(ExportConstantsTest.class, GOLDEN_BACKUP_KINDS_FILENAME);
-    final Pattern stripComments = Pattern.compile("\\s*#.*$");
-    List<String> goldenKinds = FluentIterable
-        .from(Splitter.on('\n').split(
-            Resources.toString(goldenBackupKindsResource, UTF_8).trim()))
-        .transform(
-            new Function<String, String>() {
-              @Override @Nullable public String apply(@Nullable String line) {
-                return stripComments.matcher(line).replaceFirst("");
-              }})
-        .toList();
+    List<String> goldenKinds = extractListFromFile(GOLDEN_BACKUP_KINDS_FILENAME);
     ImmutableSet<String> actualKinds = ExportConstants.getBackupKinds();
-    String updateInstructions = String.format(
-        UPDATE_INSTRUCTIONS_TEMPLATE,
-        goldenBackupKindsResource.toString(),
-        Joiner.on('\n').join(actualKinds));
+    String updateInstructions =
+        getUpdateInstructions("backed-up", goldenBackupKindsResource.toString(), actualKinds);
     assertWithMessage(updateInstructions)
         .that(actualKinds)
         .containsExactlyElementsIn(goldenKinds)
         .inOrder();
   }
 
+  @Test
+  public void testReportingKinds_matchGoldenReportingKindsFile() throws Exception {
+    URL goldenReportingKindsResource =
+        getResource(ExportConstantsTest.class, GOLDEN_REPORTING_KINDS_FILENAME);
+    List<String> goldenReportingKinds = extractListFromFile(GOLDEN_REPORTING_KINDS_FILENAME);
+    ImmutableSet<String> actualKinds = ExportConstants.getReportingKinds();
+    String updateInstructions =
+        getUpdateInstructions("reporting", goldenReportingKindsResource.toString(), actualKinds);
+    assertWithMessage(updateInstructions)
+        .that(actualKinds)
+        .containsExactlyElementsIn(goldenReportingKinds)
+        .inOrder();
+  }
+
   @Test
   public void testReportingKinds_areSubsetOfBackupKinds() throws Exception {
     assertThat(ExportConstants.getBackupKinds()).containsAllIn(ExportConstants.getReportingKinds());
   }
 
-  @Test
-  public void testReportingEntityClasses_areAllBaseEntityClasses() throws Exception {
-    for (Class<? extends ImmutableObject> clazz : ExportConstants.REPORTING_ENTITY_CLASSES) {
-      assertThat(clazz.isAnnotationPresent(Entity.class))
-          .named(String.format("class %s is an @Entity", clazz.getSimpleName()))
-          .isTrue();
+  /**
+   * Helper method to get update instructions
+   *
+   * @param name - type of entity
+   * @param resource - Resource file contents
+   * @param actualKinds - data from ExportConstants
+   * @return String of update instructions
+   */
+  private static String getUpdateInstructions(
+      String name, String resource, ImmutableSet<String> actualKinds) {
+    return String.format(
+        UPDATE_INSTRUCTIONS_TEMPLATE, name, resource, Joiner.on('\n').join(actualKinds));
   }
+
+  /**
+   * Helper method to extract list from file
+   *
+   * @param filename
+   * @return ImmutableList<String>
+   */
+  private static ImmutableList<String> extractListFromFile(String filename) {
+    String fileContents = readResourceUtf8(ExportConstantsTest.class, filename);
+    final Pattern stripComments = Pattern.compile("\\s*#.*$");
+    return FluentIterable.from(Splitter.on('\n').split(fileContents.trim()))
+        .transform(
+            new Function<String, String>() {
+              @Override
+              @Nullable
+              public String apply(@Nullable String line) {
+                return stripComments.matcher(line).replaceFirst("");
+              }
+            })
+        .toList();
   }
 }

javatests/google/registry/export/reporting_kinds.txt

@@ -0,0 +1,21 @@
+Cancellation
+ContactResource
+DomainApplicationIndex
+DomainBase
+EppResourceIndex
+ForeignKeyContactIndex
+ForeignKeyDomainIndex
+ForeignKeyHostIndex
+HistoryEntry
+HostResource
+LrpTokenEntity
+Modification
+OneTime
+PremiumList
+PremiumListEntry
+Recurring
+Registrar
+RegistrarContact
+RegistrarCredit
+RegistrarCreditBalance
+Registry