mirror of
https://github.com/google/nomulus.git
synced 2025-07-25 12:08:36 +02:00
Label classes to delete after migration - Batch 1 (#1460)
* Label classes to delete after migration - Batch 1 * Format Fix
This commit is contained in:
parent
99be1c2859
commit
bde9041600
49 changed files with 137 additions and 32 deletions
|
@ -21,6 +21,7 @@ import com.google.common.collect.AbstractIterator;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
@ -28,6 +29,7 @@ import java.io.OutputStream;
|
|||
import java.util.Iterator;
|
||||
|
||||
/** Utilities for working with backups. */
|
||||
@DeleteAfterMigration
|
||||
public class BackupUtils {
|
||||
|
||||
/** Keys for user metadata fields on commit log files in GCS. */
|
||||
|
|
|
@ -22,6 +22,7 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
|||
|
||||
import com.google.common.collect.ImmutableMultimap;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import google.registry.request.Action;
|
||||
|
@ -48,6 +49,7 @@ import org.joda.time.DateTime;
|
|||
method = Action.Method.GET,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public final class CommitLogCheckpointAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
|
|
@ -23,6 +23,7 @@ import static google.registry.util.DateTimeUtils.earliestOf;
|
|||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
|
@ -36,14 +37,14 @@ import org.joda.time.DateTime;
|
|||
/**
|
||||
* Implementation of the procedure for determining point-in-time consistent commit log checkpoint.
|
||||
*
|
||||
* <p>This algorithm examines the recently written commit log data and uses a dual-read approach
|
||||
* to determine a point-in-time consistent set of checkpoint times for the commit log buckets. By
|
||||
* <p>This algorithm examines the recently written commit log data and uses a dual-read approach to
|
||||
* determine a point-in-time consistent set of checkpoint times for the commit log buckets. By
|
||||
* "consistent" we mean, generally speaking, that if the Datastore were restored by replaying all
|
||||
* the commit logs up to the checkpoint times of the buckets, the result would be transactionally
|
||||
* correct; there must be no "holes" where restored state depends on non-restored state.
|
||||
*
|
||||
* <p>The consistency guarantee really has two parts, only one of which is provided by this
|
||||
* algorithm. The procedure below guarantees only that if the resulting checkpoint includes any
|
||||
* algorithm. The procedure below guarantees only that if the resulting checkpoint includes any
|
||||
* given commit log, it will also include all the commit logs that were both 1) actually written
|
||||
* before that commit log "in real life", and 2) have an earlier timestamp than that commit log.
|
||||
* (These criteria do not necessarily imply each other, due to the lack of a global shared clock.)
|
||||
|
@ -51,8 +52,8 @@ import org.joda.time.DateTime;
|
|||
* that depends on state from a previous transaction does indeed have a later timestamp.
|
||||
*
|
||||
* <h2>Procedure description</h2>
|
||||
* <pre>
|
||||
* {@code
|
||||
*
|
||||
* <pre>{@code
|
||||
* ComputeCheckpoint() -> returns a set consisting of a timestamp c(b_i) for every bucket b_i
|
||||
*
|
||||
* 1) read off the latest commit timestamp t(b_i) for every bucket b_i
|
||||
|
@ -63,35 +64,29 @@ import org.joda.time.DateTime;
|
|||
* a) if S is empty, let T* = +∞ (or the "end of time")
|
||||
* b) else, let T* = T - Δ, for T = min(S) and some small Δ > 0
|
||||
* 4) return the set given by: min(t(b_i), T*) for all b_i
|
||||
* }
|
||||
* </pre>
|
||||
* }</pre>
|
||||
*
|
||||
* <h2>Correctness proof of algorithm</h2>
|
||||
*
|
||||
* <p>{@literal
|
||||
* As described above, the algorithm is correct as long as it can ensure the following: given a
|
||||
* commit log X written at time t(X) to bucket b_x, and another commit log Y that was written "in
|
||||
* real life" before X and for which t(Y) < t(X), then if X is included in the checkpoint, so is Y;
|
||||
* that is, t(X) <= c(b_x) implies t(Y) <= c(b_y).
|
||||
* }
|
||||
* <p>{@literal As described above, the algorithm is correct as long as it can ensure the following:
|
||||
* given a commit log X written at time t(X) to bucket b_x, and another commit log Y that was
|
||||
* written "in real life" before X and for which t(Y) < t(X), then if X is included in the
|
||||
* checkpoint, so is Y; that is, t(X) <= c(b_x) implies t(Y) <= c(b_y). }
|
||||
*
|
||||
* <p>{@literal
|
||||
* To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e. every commit
|
||||
* log included in the checkpoint must have been seen in the first pass. Hence if X was included,
|
||||
* then X must have been written by the time we started the second pass. But since Y was written
|
||||
* "in real life" prior to X, we must have seen Y by the second pass too.
|
||||
* }
|
||||
* <p>{@literal To prove this, first note that we always have c(b_i) <= t(b_i) for every b_i, i.e.
|
||||
* every commit log included in the checkpoint must have been seen in the first pass. Hence if X was
|
||||
* included, then X must have been written by the time we started the second pass. But since Y was
|
||||
* written "in real life" prior to X, we must have seen Y by the second pass too. }
|
||||
*
|
||||
* <p>{@literal
|
||||
* Now assume towards a contradiction that X is indeed included but Y is not, i.e. that we have
|
||||
* t(X) <= c(b_x) but t(Y) > c(b_y). If Y was seen in the first pass, i.e. t(Y) <= t(b_y), then by
|
||||
* our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y). By the definition of
|
||||
* c(b_y) it must then equal T*, so we have T* < t(Y). However, this is a contradiction since
|
||||
* t(Y) < t(X) and t(X) <= c(b_x) <= T*. If instead Y was seen in the second pass but not the
|
||||
* first, t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by
|
||||
* definition, we again reach the contradiction T* < t(Y).
|
||||
* }
|
||||
* <p>{@literal Now assume towards a contradiction that X is indeed included but Y is not, i.e. that
|
||||
* we have t(X) <= c(b_x) but t(Y) > c(b_y). If Y was seen in the first pass, i.e. t(Y) <= t(b_y),
|
||||
* then by our assumption c(b_y) < t(Y) <= t(b_y), and therefore c(b_y) != t(b_y). By the definition
|
||||
* of c(b_y) it must then equal T*, so we have T* < t(Y). However, this is a contradiction since
|
||||
* t(Y) < t(X) and t(X) <= c(b_x) <= T*. If instead Y was seen in the second pass but not the first,
|
||||
* t'(b_y) exists and we must have t'(b_y) <= t(Y), but then since T* < T <= t'(b_y) by definition,
|
||||
* we again reach the contradiction T* < t(Y). }
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
class CommitLogCheckpointStrategy {
|
||||
|
||||
@Inject Ofy ofy;
|
||||
|
|
|
@ -20,6 +20,7 @@ import static google.registry.backup.BackupUtils.createDeserializingIterator;
|
|||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
|
@ -38,6 +39,7 @@ import java.util.Iterator;
|
|||
* <p>This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial
|
||||
* population of the Cloud SQL database.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public final class CommitLogImports {
|
||||
|
||||
private CommitLogImports() {}
|
||||
|
|
|
@ -35,6 +35,7 @@ import google.registry.mapreduce.MapreduceRunner;
|
|||
import google.registry.mapreduce.inputs.CommitLogManifestInput;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
|
||||
|
@ -68,6 +69,7 @@ import org.joda.time.Duration;
|
|||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
@DeleteAfterMigration
|
||||
public final class DeleteOldCommitLogsAction implements Runnable {
|
||||
|
||||
private static final int NUM_MAP_SHARDS = 20;
|
||||
|
|
|
@ -20,10 +20,12 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto;
|
|||
import com.google.storage.onestore.v3.OnestoreEntity.Path;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning;
|
||||
import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Objects;
|
||||
|
||||
/** Utilities for handling imported Datastore entities. */
|
||||
@DeleteAfterMigration
|
||||
public class EntityImports {
|
||||
|
||||
/**
|
||||
|
|
|
@ -40,6 +40,7 @@ import com.googlecode.objectify.Key;
|
|||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
|
@ -63,6 +64,7 @@ import org.joda.time.DateTime;
|
|||
method = Action.Method.POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public final class ExportCommitLogDiffAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
|
|
@ -33,6 +33,7 @@ import com.google.common.util.concurrent.ListeningExecutorService;
|
|||
import com.google.common.util.concurrent.UncheckedExecutionException;
|
||||
import google.registry.backup.BackupModule.Backups;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
|
@ -44,6 +45,7 @@ import javax.inject.Provider;
|
|||
import org.joda.time.DateTime;
|
||||
|
||||
/** Utility class to list commit logs diff files stored on GCS. */
|
||||
@DeleteAfterMigration
|
||||
class GcsDiffFileLister {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
|
|
@ -34,6 +34,7 @@ import com.google.common.flogger.FluentLogger;
|
|||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.UpdateAutoTimestamp;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
|
||||
import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
|
||||
|
@ -68,6 +69,7 @@ import org.joda.time.Seconds;
|
|||
method = Method.POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class ReplayCommitLogsToSqlAction implements Runnable {
|
||||
|
||||
static final String PATH = "/_dr/task/replayCommitLogsToSql";
|
||||
|
|
|
@ -37,6 +37,7 @@ import google.registry.config.RegistryConfig.Config;
|
|||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
|
@ -64,6 +65,7 @@ import org.joda.time.DateTime;
|
|||
method = Action.Method.POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class RestoreCommitLogsAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
|
|
@ -19,6 +19,7 @@ import com.google.appengine.api.datastore.EntityTranslator;
|
|||
import com.google.appengine.api.datastore.Key;
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.auto.value.extension.memoized.Memoized;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import google.registry.model.ofy.CommitLogMutation;
|
||||
import java.io.Serializable;
|
||||
|
@ -76,6 +77,7 @@ import javax.annotation.Nullable;
|
|||
* property type in this class.
|
||||
*/
|
||||
@AutoValue
|
||||
@DeleteAfterMigration
|
||||
public abstract class VersionedEntity implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
|
|
@ -73,6 +73,7 @@ import google.registry.mapreduce.inputs.EppResourceInputs;
|
|||
import google.registry.mapreduce.inputs.NullInput;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.annotations.ExternalMessagingName;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
|
@ -114,6 +115,7 @@ import org.joda.time.Duration;
|
|||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/deleteContactsAndHosts",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class DeleteContactsAndHostsAction implements Runnable {
|
||||
|
||||
static final String KIND_CONTACT = getKind(ContactResource.class);
|
||||
|
|
|
@ -24,6 +24,7 @@ import com.googlecode.objectify.Key;
|
|||
import google.registry.mapreduce.MapreduceRunner;
|
||||
import google.registry.mapreduce.inputs.EppResourceInputs;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Parameter;
|
||||
import google.registry.request.Response;
|
||||
|
@ -56,6 +57,7 @@ import javax.inject.Inject;
|
|||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
// No longer needed in SQL. Subject to future removal.
|
||||
@Deprecated
|
||||
@DeleteAfterMigration
|
||||
public class ResaveAllEppResourcesAction implements Runnable {
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
|
|
|
@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableSet;
|
|||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.Response;
|
||||
import google.registry.request.auth.Auth;
|
||||
|
@ -45,6 +46,7 @@ import javax.inject.Inject;
|
|||
service = Action.Service.BACKEND,
|
||||
path = "/_dr/task/wipeOutDatastore",
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class WipeoutDatastoreAction implements Runnable {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
|
|
|
@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableSet;
|
|||
import com.googlecode.objectify.Key;
|
||||
import google.registry.backup.VersionedEntity;
|
||||
import google.registry.beam.initsql.Transforms;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.contact.ContactHistory;
|
||||
|
@ -51,6 +52,7 @@ import org.apache.beam.sdk.values.TupleTagList;
|
|||
import org.joda.time.DateTime;
|
||||
|
||||
/** Utilities for loading Datastore snapshots. */
|
||||
@DeleteAfterMigration
|
||||
public final class DatastoreSnapshots {
|
||||
|
||||
private DatastoreSnapshots() {}
|
||||
|
|
|
@ -24,6 +24,7 @@ import google.registry.config.RegistryConfig;
|
|||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.config.RegistryConfig.ConfigModule;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.util.Clock;
|
||||
import google.registry.util.UtilsModule;
|
||||
import java.io.IOException;
|
||||
|
@ -37,6 +38,7 @@ import org.joda.time.Instant;
|
|||
import org.joda.time.Interval;
|
||||
|
||||
/** Finds the necessary information for loading the most recent Datastore snapshot. */
|
||||
@DeleteAfterMigration
|
||||
public class LatestDatastoreSnapshotFinder {
|
||||
private final String projectId;
|
||||
private final GcsUtils gcsUtils;
|
||||
|
|
|
@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableSet;
|
|||
import com.google.common.collect.ImmutableSetMultimap;
|
||||
import com.google.common.collect.Streams;
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.bulkquery.BulkQueryEntities;
|
||||
import google.registry.model.bulkquery.DomainBaseLite;
|
||||
|
@ -76,6 +77,7 @@ import org.apache.beam.sdk.values.TypeDescriptors;
|
|||
* contains optimizations specifically for the production database at the current size, e.g.,
|
||||
* parallel queries for select tables.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public final class SqlSnapshots {
|
||||
|
||||
private SqlSnapshots() {}
|
||||
|
|
|
@ -25,6 +25,7 @@ import google.registry.beam.common.DatabaseSnapshot;
|
|||
import google.registry.beam.common.RegistryPipelineWorkerInitializer;
|
||||
import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
|
||||
import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.domain.DomainHistory;
|
||||
import google.registry.model.replay.SqlEntity;
|
||||
|
@ -54,6 +55,7 @@ import org.joda.time.Duration;
|
|||
* Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
|
||||
* (secondary storage).
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class ValidateSqlPipeline {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
|
|
|
@ -15,6 +15,8 @@
|
|||
package google.registry.beam.comparedb;
|
||||
|
||||
import google.registry.beam.common.RegistryPipelineOptions;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
|
||||
/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
|
||||
@DeleteAfterMigration
|
||||
public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {}
|
||||
|
|
|
@ -26,6 +26,7 @@ import google.registry.config.RegistryEnvironment;
|
|||
import google.registry.model.BackupGroupRoot;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.contact.ContactBase;
|
||||
import google.registry.model.contact.ContactHistory;
|
||||
|
@ -52,6 +53,7 @@ import org.apache.beam.sdk.values.KV;
|
|||
import org.apache.beam.sdk.values.TupleTag;
|
||||
|
||||
/** Helpers for use by {@link ValidateSqlPipeline}. */
|
||||
@DeleteAfterMigration
|
||||
final class ValidateSqlUtils {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
|
|
|
@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableSortedSet;
|
|||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.datastore.v1.Entity;
|
||||
import google.registry.config.RegistryEnvironment;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import org.apache.beam.sdk.Pipeline;
|
||||
|
@ -79,6 +80,7 @@ import org.apache.beam.sdk.values.TupleTagList;
|
|||
* types in the Datastore using the {@code --numOfKindsHint} argument. If the default value for this
|
||||
* parameter is too low, performance will suffer.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class BulkDeleteDatastorePipeline {
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
|
|
|
@ -53,6 +53,7 @@ import com.google.datastore.v1.client.DatastoreOptions;
|
|||
import com.google.datastore.v1.client.QuerySplitter;
|
||||
import com.google.protobuf.Int32Value;
|
||||
import com.google.rpc.Code;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.NoSuchElementException;
|
||||
|
@ -80,6 +81,7 @@ import org.joda.time.Duration;
|
|||
* Contains an adaptation of {@link org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read}. See
|
||||
* {@link MultiRead} for details.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class DatastoreV1 {
|
||||
|
||||
// A package-private constructor to prevent direct instantiation from outside of this package
|
||||
|
|
|
@ -21,12 +21,14 @@ import static com.google.common.base.Strings.isNullOrEmpty;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
/**
|
||||
* Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a
|
||||
* Datastore export and Nomulus CommitLogs that overlap with the export.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public final class BackupPaths {
|
||||
|
||||
private BackupPaths() {}
|
||||
|
|
|
@ -18,9 +18,11 @@ import static com.google.common.base.Preconditions.checkArgument;
|
|||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
|
||||
import com.google.appengine.api.datastore.Entity;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Objects;
|
||||
|
||||
/** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */
|
||||
@DeleteAfterMigration
|
||||
final class DomainBaseUtil {
|
||||
|
||||
private DomainBaseUtil() {}
|
||||
|
|
|
@ -23,6 +23,7 @@ import com.googlecode.objectify.Key;
|
|||
import google.registry.backup.VersionedEntity;
|
||||
import google.registry.beam.common.RegistryJpaIO;
|
||||
import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.billing.BillingEvent;
|
||||
import google.registry.model.common.Cursor;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
|
@ -94,6 +95,7 @@ import org.joda.time.DateTime;
|
|||
* may start writing {@code DomainBase} entities before all {@code Registry}, {@code Registrar} and
|
||||
* {@code ContactResource} entities have been persisted.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class InitSqlPipeline implements Serializable {
|
||||
|
||||
/**
|
||||
|
|
|
@ -15,10 +15,12 @@
|
|||
package google.registry.beam.initsql;
|
||||
|
||||
import google.registry.beam.common.RegistryPipelineOptions;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import org.apache.beam.sdk.options.Description;
|
||||
import org.apache.beam.sdk.options.Validation;
|
||||
|
||||
/** Pipeline options for {@link InitSqlPipeline} */
|
||||
@DeleteAfterMigration
|
||||
public interface InitSqlPipelineOptions extends RegistryPipelineOptions {
|
||||
|
||||
@Description("The root directory of the export to load.")
|
||||
|
|
|
@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableSet;
|
|||
import com.google.common.collect.Streams;
|
||||
import google.registry.backup.CommitLogImports;
|
||||
import google.registry.backup.VersionedEntity;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.billing.BillingEvent.Flag;
|
||||
import google.registry.model.billing.BillingEvent.Reason;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
|
@ -80,6 +81,7 @@ import org.joda.time.DateTime;
|
|||
* {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export
|
||||
* files and Nomulus CommitLog files.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public final class Transforms {
|
||||
|
||||
private Transforms() {}
|
||||
|
|
|
@ -21,12 +21,14 @@ import com.google.common.collect.ImmutableSet;
|
|||
import com.google.common.collect.Ordering;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.EntityClasses;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.annotations.InCrossTld;
|
||||
import google.registry.model.annotations.NotBackedUp;
|
||||
import google.registry.model.annotations.ReportedOn;
|
||||
import google.registry.model.annotations.VirtualEntity;
|
||||
|
||||
/** Constants related to export code. */
|
||||
@DeleteAfterMigration
|
||||
public final class AnnotatedEntities {
|
||||
|
||||
/** Returns the names of kinds to include in Datastore backups. */
|
||||
|
|
|
@ -21,6 +21,7 @@ import com.google.common.flogger.FluentLogger;
|
|||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.export.datastore.DatastoreAdmin;
|
||||
import google.registry.export.datastore.Operation;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.request.Response;
|
||||
|
@ -46,6 +47,7 @@ import javax.inject.Inject;
|
|||
method = POST,
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class BackupDatastoreAction implements Runnable {
|
||||
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
|
|
@ -33,6 +33,7 @@ import com.google.common.collect.Sets;
|
|||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.export.datastore.DatastoreAdmin;
|
||||
import google.registry.export.datastore.Operation;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
|
@ -60,6 +61,7 @@ import org.joda.time.format.PeriodFormat;
|
|||
method = {POST, GET},
|
||||
automaticallyPrintOk = true,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class CheckBackupAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into this action. */
|
||||
|
|
|
@ -26,6 +26,7 @@ import com.google.appengine.api.taskqueue.TaskOptions.Method;
|
|||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.bigquery.CheckedBigquery;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.request.Parameter;
|
||||
|
@ -40,6 +41,7 @@ import javax.inject.Inject;
|
|||
path = UpdateSnapshotViewAction.PATH,
|
||||
method = POST,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class UpdateSnapshotViewAction implements Runnable {
|
||||
|
||||
/** Headers for passing parameters into the servlet. */
|
||||
|
|
|
@ -39,6 +39,7 @@ import google.registry.bigquery.BigqueryUtils.WriteDisposition;
|
|||
import google.registry.bigquery.CheckedBigquery;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.request.Action;
|
||||
import google.registry.request.HttpException.BadRequestException;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
|
@ -53,6 +54,7 @@ import javax.inject.Inject;
|
|||
path = UploadDatastoreBackupAction.PATH,
|
||||
method = POST,
|
||||
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
|
||||
@DeleteAfterMigration
|
||||
public class UploadDatastoreBackupAction implements Runnable {
|
||||
|
||||
/** Parameter names for passing parameters into the servlet. */
|
||||
|
|
|
@ -26,6 +26,7 @@ import com.google.api.client.json.JsonFactory;
|
|||
import com.google.api.client.util.Key;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Collection;
|
||||
import java.util.Optional;
|
||||
|
||||
|
@ -33,6 +34,7 @@ import java.util.Optional;
|
|||
* Java client to <a href="https://cloud.google.com/datastore/docs/reference/admin/rest/">Cloud
|
||||
* Datastore Admin REST API</a>.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class DatastoreAdmin extends AbstractGoogleJsonClient {
|
||||
|
||||
private static final String ROOT_URL = "https://datastore.googleapis.com/v1/";
|
||||
|
|
|
@ -18,11 +18,13 @@ import dagger.Module;
|
|||
import dagger.Provides;
|
||||
import google.registry.config.CredentialModule;
|
||||
import google.registry.config.RegistryConfig;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.util.GoogleCredentialsBundle;
|
||||
import javax.inject.Singleton;
|
||||
|
||||
/** Dagger module that configures provision of {@link DatastoreAdmin}. */
|
||||
@Module
|
||||
@DeleteAfterMigration
|
||||
public abstract class DatastoreAdminModule {
|
||||
|
||||
@Singleton
|
||||
|
|
|
@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
|
|||
import com.google.api.client.json.GenericJson;
|
||||
import com.google.api.client.util.Key;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
|
@ -29,6 +30,7 @@ import java.util.List;
|
|||
* <p>Please note that properties not used by Domain Registry are not included, e.g., {@code
|
||||
* namespaceIds}.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class EntityFilter extends GenericJson {
|
||||
|
||||
@Key private List<String> kinds = ImmutableList.of();
|
||||
|
|
|
@ -22,6 +22,7 @@ import com.google.api.client.util.Key;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.export.datastore.DatastoreAdmin.Get;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.util.Clock;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
@ -34,6 +35,7 @@ import org.joda.time.Duration;
|
|||
*
|
||||
* <p>{@link Operation} instances are parsed from the JSON payload in Datastore response messages.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class Operation extends GenericJson {
|
||||
|
||||
private static final String STATE_SUCCESS = "SUCCESSFUL";
|
||||
|
|
|
@ -22,12 +22,14 @@ import com.google.common.collect.ImmutableSet;
|
|||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.index.EppResourceIndexBucket;
|
||||
|
||||
/**
|
||||
* A MapReduce {@link Input} that loads all child objects of a given set of types, that are children
|
||||
* of given {@link EppResource} types.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
class ChildEntityInput<R extends EppResource, I extends ImmutableObject>
|
||||
extends EppResourceBaseInput<I> {
|
||||
|
||||
|
|
|
@ -27,6 +27,7 @@ import com.googlecode.objectify.Key;
|
|||
import com.googlecode.objectify.annotation.Entity;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.ImmutableObject;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.EppResourceIndexBucket;
|
||||
import java.io.IOException;
|
||||
|
@ -34,9 +35,10 @@ import java.util.NoSuchElementException;
|
|||
import javax.annotation.Nullable;
|
||||
|
||||
/**
|
||||
* Reader that maps over {@link EppResourceIndex} and returns resources that are children of
|
||||
* {@link EppResource} objects.
|
||||
* Reader that maps over {@link EppResourceIndex} and returns resources that are children of {@link
|
||||
* EppResource} objects.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
class ChildEntityReader<R extends EppResource, I extends ImmutableObject> extends InputReader<I> {
|
||||
|
||||
private static final long serialVersionUID = 7481761146349663848L;
|
||||
|
|
|
@ -18,6 +18,7 @@ import com.google.appengine.tools.mapreduce.Input;
|
|||
import com.google.appengine.tools.mapreduce.InputReader;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import java.util.List;
|
||||
|
@ -25,6 +26,7 @@ import javax.annotation.Nullable;
|
|||
import org.joda.time.DateTime;
|
||||
|
||||
/** Base class for {@link Input} classes that map over {@link CommitLogManifest}. */
|
||||
@DeleteAfterMigration
|
||||
public class CommitLogManifestInput extends Input<Key<CommitLogManifest>> {
|
||||
|
||||
private static final long serialVersionUID = 6744322799131602384L;
|
||||
|
|
|
@ -21,6 +21,7 @@ import com.google.appengine.api.datastore.QueryResultIterator;
|
|||
import com.google.appengine.tools.mapreduce.InputReader;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.cmd.Query;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.CommitLogBucket;
|
||||
import google.registry.model.ofy.CommitLogManifest;
|
||||
import java.util.NoSuchElementException;
|
||||
|
@ -28,6 +29,7 @@ import javax.annotation.Nullable;
|
|||
import org.joda.time.DateTime;
|
||||
|
||||
/** {@link InputReader} that maps over {@link CommitLogManifest}. */
|
||||
@DeleteAfterMigration
|
||||
class CommitLogManifestReader
|
||||
extends RetryingInputReader<Key<CommitLogManifest>, Key<CommitLogManifest>> {
|
||||
|
||||
|
|
|
@ -21,6 +21,7 @@ import com.google.appengine.tools.mapreduce.InputReader;
|
|||
import com.google.common.collect.ImmutableSet;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.index.EppResourceIndexBucket;
|
||||
|
||||
/**
|
||||
|
@ -28,6 +29,7 @@ import google.registry.model.index.EppResourceIndexBucket;
|
|||
*
|
||||
* <p>When mapping over keys we can't distinguish between Objectify polymorphic types.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
class EppResourceKeyInput<R extends EppResource> extends EppResourceBaseInput<Key<R>> {
|
||||
|
||||
private static final long serialVersionUID = -5426821384707653743L;
|
||||
|
|
|
@ -18,6 +18,7 @@ import com.google.appengine.tools.mapreduce.InputReader;
|
|||
import com.google.common.collect.ImmutableSet;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.index.EppResourceIndex;
|
||||
import google.registry.model.index.EppResourceIndexBucket;
|
||||
import java.util.NoSuchElementException;
|
||||
|
@ -27,6 +28,7 @@ import java.util.NoSuchElementException;
|
|||
*
|
||||
* <p>When mapping over keys we can't distinguish between Objectify polymorphic types.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
class EppResourceKeyReader<R extends EppResource> extends EppResourceBaseReader<Key<R>> {
|
||||
|
||||
private static final long serialVersionUID = -428232054739189774L;
|
||||
|
|
|
@ -23,6 +23,7 @@ import com.google.appengine.api.datastore.QueryResultIterator;
|
|||
import com.google.appengine.tools.mapreduce.InputReader;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.cmd.Query;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.util.SystemSleeper;
|
||||
import java.util.NoSuchElementException;
|
||||
|
@ -40,6 +41,7 @@ import javax.annotation.Nullable;
|
|||
*
|
||||
* <p>I is the internal Objectify read type, while T is the InputReader return type.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
abstract class RetryingInputReader<I, T> extends InputReader<T> {
|
||||
|
||||
private static final long serialVersionUID = -4897677478541818899L;
|
||||
|
|
|
@ -17,6 +17,7 @@ package google.registry.model;
|
|||
import com.google.apphosting.api.ApiProxy;
|
||||
import com.google.apphosting.api.ApiProxy.Environment;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.model.annotations.DeleteAfterMigration;
|
||||
import google.registry.model.ofy.ObjectifyService;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.lang.reflect.Method;
|
||||
|
@ -39,6 +40,7 @@ import java.lang.reflect.Proxy;
|
|||
* <p>Note that conversion from Objectify objects to Datastore {@code Entities} still requires the
|
||||
* Datastore service.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
public class AppEngineEnvironment {
|
||||
|
||||
private Environment environment;
|
||||
|
|
|
@ -0,0 +1,20 @@
|
|||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.model.annotations;
|
||||
|
||||
/**
|
||||
* Annotation to indicate a class that should be deleted after the database migration is complete.
|
||||
*/
|
||||
public @interface DeleteAfterMigration {}
|
|
@ -28,6 +28,7 @@ import java.lang.annotation.Target;
|
|||
* <p>This means that the entity's <code>@Parent</code> field has to have the value of {@link
|
||||
* EntityGroupRoot#getCrossTldKey}.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ElementType.TYPE})
|
||||
@Inherited
|
||||
|
|
|
@ -24,6 +24,7 @@ import java.lang.annotation.Target;
|
|||
* Annotation for an Objectify {@link Entity} to indicate that it should not be backed up by the
|
||||
* default Datastore backup configuration (it may be backed up by something else).
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target(ElementType.TYPE)
|
||||
public @interface NotBackedUp {
|
||||
|
|
|
@ -23,6 +23,7 @@ import java.lang.annotation.Target;
|
|||
/**
|
||||
* Annotation for an Objectify {@link Entity} to indicate that it should be exported to BigQuery.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target({ElementType.TYPE})
|
||||
public @interface ReportedOn {}
|
||||
|
|
|
@ -23,9 +23,10 @@ import java.lang.annotation.Target;
|
|||
/**
|
||||
* Annotation for an Objectify {@link Entity} to indicate that it is a "virtual entity".
|
||||
*
|
||||
* <p>A virtual entity type exists only to define part of the parentage key hierarchy for its
|
||||
* child entities, and is never actually persisted and thus has no fields besides its ID field.
|
||||
* <p>A virtual entity type exists only to define part of the parentage key hierarchy for its child
|
||||
* entities, and is never actually persisted and thus has no fields besides its ID field.
|
||||
*/
|
||||
@DeleteAfterMigration
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Target(ElementType.TYPE)
|
||||
public @interface VirtualEntity {}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue