diff --git a/core/build.gradle b/core/build.gradle index f918c353d..79f65dc0e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -765,11 +765,6 @@ createUberJar( // User should install gcloud and login to GCP before invoking this tasks. if (environment == 'alpha') { def pipelines = [ - initSql : - [ - mainClass: 'google.registry.beam.initsql.InitSqlPipeline', - metaData : 'google/registry/beam/init_sql_pipeline_metadata.json' - ], bulkDeleteDatastore: [ mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline', @@ -888,48 +883,6 @@ task buildToolImage(dependsOn: nomulus, type: Exec) { commandLine 'docker', 'build', '-t', 'nomulus-tool', '.' } -task generateInitSqlPipelineGraph(type: FilteringTest) { - tests = ['InitSqlPipelineGraphTest.createPipeline_compareGraph'] - ignoreFailures = true -} - -task updateInitSqlPipelineGraph(type: Copy) { - def graphRelativePath = 'google/registry/beam/initsql/' - from ("${projectDir}/build/resources/test/${graphRelativePath}") { - include 'pipeline_curr.dot' - rename 'curr', 'golden' - } - into "src/test/resources/${graphRelativePath}" - - dependsOn generateInitSqlPipelineGraph - - doLast { - if (com.google.common.base.Strings.isNullOrEmpty(project.dot_path)) { - getLogger().info('Property dot_path is null. Not creating image for pipeline graph.') - } - def dotPath = project.dot_path - if (!new File(dotPath).exists()) { - throw new RuntimeException( - """\ - ${dotPath} not found. Make sure graphviz is installed - and the dot_path property is set correctly.""" - .stripIndent()) - } - def goldenGraph = "src/test/resources/${graphRelativePath}/pipeline_golden.dot" - def goldenImage = "src/test/resources/${graphRelativePath}/pipeline_golden.png" - def cmd = "${dotPath} -Tpng -o \"${goldenImage}\" \"${goldenGraph}\"" - try { - rootProject.ext.execInBash(cmd, projectDir) - } catch (Throwable throwable) { - throw new RuntimeException( - """\ - Failed to generate golden image with command ${cmd} - Error: ${throwable.getMessage()} - """) - } - } -} - // Build the devtool jar. createUberJar( 'devtool', diff --git a/core/src/main/java/google/registry/backup/CommitLogImports.java b/core/src/main/java/google/registry/backup/CommitLogImports.java deleted file mode 100644 index 9c98cdee6..000000000 --- a/core/src/main/java/google/registry/backup/CommitLogImports.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.backup; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static google.registry.backup.BackupUtils.createDeserializingIterator; - -import com.google.common.collect.ImmutableList; -import google.registry.model.ImmutableObject; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.ofy.CommitLogCheckpoint; -import google.registry.model.ofy.CommitLogManifest; -import google.registry.model.ofy.CommitLogMutation; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.util.Iterator; - -/** - * Helpers for reading CommitLog records from a file. - * - *

This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial - * population of the Cloud SQL database. - */ -@DeleteAfterMigration -public final class CommitLogImports { - - private CommitLogImports() {} - - /** - * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link - * ImmutableList} of {@link ImmutableList}s of {@link VersionedEntity} records where the inner - * lists each consist of one transaction. Upon completion the {@code inputStream} is closed. - * - *

The returned list may be empty, since CommitLogs are written at fixed intervals regardless - * if actual changes exist. Each sublist, however, will not be empty. - * - *

A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated) - * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence - * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted - * entity keys, whereas each {@code CommitLogMutation} contains one whole entity. - */ - static ImmutableList> loadEntitiesByTransaction( - InputStream inputStream) { - try (InputStream input = new BufferedInputStream(inputStream)) { - Iterator commitLogs = createDeserializingIterator(input, false); - checkState(commitLogs.hasNext()); - checkState(commitLogs.next() instanceof CommitLogCheckpoint); - - ImmutableList.Builder> resultBuilder = - new ImmutableList.Builder<>(); - ImmutableList.Builder currentTransactionBuilder = - new ImmutableList.Builder<>(); - - while (commitLogs.hasNext()) { - ImmutableObject currentObject = commitLogs.next(); - if (currentObject instanceof CommitLogManifest) { - // CommitLogManifest means we are starting a new transaction - addIfNonempty(resultBuilder, currentTransactionBuilder); - currentTransactionBuilder = new ImmutableList.Builder<>(); - VersionedEntity.fromManifest((CommitLogManifest) currentObject) - .forEach(currentTransactionBuilder::add); - } else if (currentObject instanceof CommitLogMutation) { - currentTransactionBuilder.add( - VersionedEntity.fromMutation((CommitLogMutation) currentObject)); - } else { - throw new IllegalStateException( - String.format("Unknown entity type %s in commit logs", currentObject.getClass())); - } - } - // Add the last transaction in (if it's not empty) - addIfNonempty(resultBuilder, currentTransactionBuilder); - return resultBuilder.build(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link - * ImmutableList} of {@link VersionedEntity} records. Upon completion the {@code inputStream} is - * closed. - * - *

The returned list may be empty, since CommitLogs are written at fixed intervals regardless - * if actual changes exist. - * - *

A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated) - * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence - * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted - * entity keys, whereas each {@code CommitLogMutation} contains one whole entity. - */ - static ImmutableList loadEntities(InputStream inputStream) { - return loadEntitiesByTransaction(inputStream).stream() - .flatMap(ImmutableList::stream) - .collect(toImmutableList()); - } - - /** Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link File}. */ - public static ImmutableList loadEntities(File commitLogFile) { - try { - return loadEntities(new FileInputStream(commitLogFile)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link - * ReadableByteChannel}. - */ - public static ImmutableList loadEntities(ReadableByteChannel channel) { - return loadEntities(Channels.newInputStream(channel)); - } - - private static void addIfNonempty( - ImmutableList.Builder> resultBuilder, - ImmutableList.Builder currentTransactionBuilder) { - ImmutableList currentTransaction = currentTransactionBuilder.build(); - if (!currentTransaction.isEmpty()) { - resultBuilder.add(currentTransaction); - } - } -} diff --git a/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java b/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java deleted file mode 100644 index 947250076..000000000 --- a/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.batch; - -import static com.google.appengine.api.taskqueue.QueueConstants.maxLeaseCount; -import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static google.registry.batch.AsyncTaskEnqueuer.PARAM_HOST_KEY; -import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME; -import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME; -import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH; -import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; -import static google.registry.model.EppResourceUtils.getLinkedDomainKeys; -import static google.registry.model.EppResourceUtils.isActive; -import static google.registry.model.EppResourceUtils.isDeleted; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.util.DateTimeUtils.latestOf; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.logging.Level.INFO; -import static java.util.logging.Level.SEVERE; -import static org.joda.time.Duration.standardHours; - -import com.google.appengine.api.taskqueue.LeaseOptions; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.TaskHandle; -import com.google.appengine.api.taskqueue.TransientFailureException; -import com.google.appengine.tools.mapreduce.Mapper; -import com.google.appengine.tools.mapreduce.Reducer; -import com.google.appengine.tools.mapreduce.ReducerInput; -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.flogger.FluentLogger; -import google.registry.batch.AsyncTaskMetrics.OperationResult; -import google.registry.dns.DnsQueue; -import google.registry.mapreduce.MapreduceRunner; -import google.registry.mapreduce.inputs.NullInput; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.domain.DomainBase; -import google.registry.model.host.HostResource; -import google.registry.model.server.Lock; -import google.registry.persistence.VKey; -import google.registry.request.Action; -import google.registry.request.Response; -import google.registry.request.auth.Auth; -import google.registry.util.Clock; -import google.registry.util.NonFinalForTesting; -import google.registry.util.RequestStatusChecker; -import google.registry.util.Retrier; -import google.registry.util.SystemClock; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.logging.Level; -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.inject.Named; -import org.apache.http.HttpStatus; -import org.joda.time.DateTime; -import org.joda.time.Duration; - -/** Performs batched DNS refreshes for applicable domains following a host rename. */ -@Action( - service = Action.Service.BACKEND, - path = "/_dr/task/refreshDnsOnHostRename", - auth = Auth.AUTH_INTERNAL_OR_ADMIN) -@DeleteAfterMigration -@Deprecated -public class RefreshDnsOnHostRenameAction implements Runnable { - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final Duration LEASE_LENGTH = standardHours(4); - - @Inject AsyncTaskMetrics asyncTaskMetrics; - @Inject Clock clock; - @Inject MapreduceRunner mrRunner; - @Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue; - - @Inject DnsQueue dnsQueue; - @Inject RequestStatusChecker requestStatusChecker; - @Inject Response response; - @Inject Retrier retrier; - @Inject RefreshDnsOnHostRenameAction() {} - - @Override - public void run() { - // Check if the lock can be acquired, and if not, a previous run of this mapreduce is still - // executing, so return early. - Optional lock = - Lock.acquire( - RefreshDnsOnHostRenameAction.class.getSimpleName(), - null, - LEASE_LENGTH, - requestStatusChecker, - false); - - if (!lock.isPresent()) { - logRespondAndUnlock(INFO, "Can't acquire lock; aborting.", lock); - return; - } - - // Lease the async tasks to process. - LeaseOptions options = - LeaseOptions.Builder.withCountLimit(maxLeaseCount()) - .leasePeriod(LEASE_LENGTH.getStandardSeconds(), SECONDS); - List tasks = pullQueue.leaseTasks(options); - asyncTaskMetrics.recordDnsRefreshBatchSize(tasks.size()); - - // Check if there are no tasks to process, and if so, return early. - if (tasks.isEmpty()) { - logRespondAndUnlock( - INFO, "No DNS refresh on host rename tasks to process in pull queue; finishing.", lock); - return; - } - - ImmutableList.Builder requestsBuilder = new ImmutableList.Builder<>(); - ImmutableList.Builder> hostKeys = new ImmutableList.Builder<>(); - final List requestsToDelete = new ArrayList<>(); - - for (TaskHandle task : tasks) { - try { - DnsRefreshRequest request = DnsRefreshRequest.createFromTask(task, clock.nowUtc()); - if (request.isRefreshNeeded()) { - requestsBuilder.add(request); - hostKeys.add(request.hostKey()); - } else { - // Skip hosts that are deleted. - requestsToDelete.add(request); - } - } catch (Exception e) { - logger.atSevere().withCause(e).log( - "Could not parse DNS refresh for host request, delaying task for a day: %s", task); - // Grab the lease for a whole day, so it won't continue throwing errors every five minutes. - pullQueue.modifyTaskLease(task, 1L, DAYS); - } - } - - deleteTasksWithRetry( - requestsToDelete, pullQueue, asyncTaskMetrics, retrier, OperationResult.STALE); - ImmutableList refreshRequests = requestsBuilder.build(); - if (refreshRequests.isEmpty()) { - logRespondAndUnlock( - INFO, "No async DNS refreshes to process because all renamed hosts are deleted.", lock); - } else { - logger.atInfo().log( - "Processing asynchronous DNS refresh for renamed hosts: %s", hostKeys.build()); - if (tm().isOfy()) { - runMapreduce(refreshRequests, lock); - } else { - try { - refreshRequests.stream() - .flatMap( - request -> - getLinkedDomainKeys(request.hostKey(), request.lastUpdateTime(), null) - .stream()) - .distinct() - .map(domainKey -> tm().transact(() -> tm().loadByKey(domainKey).getDomainName())) - .forEach( - domainName -> { - retrier.callWithRetry( - () -> dnsQueue.addDomainRefreshTask(domainName), - TransientFailureException.class); - logger.atInfo().log("Enqueued DNS refresh for domain '%s'.", domainName); - }); - deleteTasksWithRetry( - refreshRequests, - getQueue(QUEUE_ASYNC_HOST_RENAME), - asyncTaskMetrics, - retrier, - OperationResult.SUCCESS); - } catch (Throwable t) { - String message = "Error refreshing DNS on host rename."; - logger.atSevere().withCause(t).log(message); - response.setPayload(message); - response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); - } finally { - lock.get().release(); - } - } - } - } - - private void runMapreduce(ImmutableList refreshRequests, Optional lock) { - try { - mrRunner - .setJobName("Enqueue DNS refreshes for domains referencing renamed hosts") - .setModuleName("backend") - .setDefaultReduceShards(1) - .runMapreduce( - new RefreshDnsOnHostRenameMapper(refreshRequests, retrier), - new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier), - // Add an extra NullInput so that the reducer always fires exactly once. - ImmutableList.of(new NullInput<>(), createEntityInput(DomainBase.class))) - .sendLinkToMapreduceConsole(response); - } catch (Throwable t) { - logRespondAndUnlock( - SEVERE, "Error starting mapreduce to refresh DNS for renamed hosts.", lock); - } - } - - private void logRespondAndUnlock(Level level, String message, Optional lock) { - logger.at(level).log(message); - response.setPayload(message); - lock.ifPresent(Lock::release); - } - - /** Map over domains and refresh the DNS of those that reference the renamed hosts. */ - public static class RefreshDnsOnHostRenameMapper - extends Mapper { - - private static final long serialVersionUID = -5261698524424335531L; - private static final DnsQueue dnsQueue = DnsQueue.create(); - - private final ImmutableList refreshRequests; - private final Retrier retrier; - - RefreshDnsOnHostRenameMapper( - ImmutableList refreshRequests, Retrier retrier) { - this.refreshRequests = refreshRequests; - this.retrier = retrier; - } - - @Override - public final void map(@Nullable final DomainBase domain) { - if (domain == null) { - // Emit a single value so that the reducer always runs. The key and value don't matter. - emit(true, true); - return; - } - VKey referencingHostKey = null; - for (DnsRefreshRequest request : refreshRequests) { - if (isActive(domain, request.lastUpdateTime()) - && domain.getNameservers().contains(request.hostKey())) { - referencingHostKey = request.hostKey(); - break; - } - } - if (referencingHostKey != null) { - retrier.callWithRetry( - () -> dnsQueue.addDomainRefreshTask(domain.getDomainName()), - TransientFailureException.class); - logger.atInfo().log( - "Enqueued DNS refresh for domain %s referenced by host %s.", - domain.getDomainName(), referencingHostKey); - getContext().incrementCounter("domains refreshed"); - } else { - getContext().incrementCounter("domains not refreshed"); - } - - // Don't catch errors -- we allow the mapreduce to terminate on any errors that can't be - // resolved by retrying the transaction. The reducer only fires if the mapper completes - // without errors, meaning that it is acceptable to delete all tasks. - } - } - - /** - * A reducer that always fires exactly once. - * - *

This is really a reducer in name only; what it's really doing is waiting for all of the - * mapper tasks to finish, and then delete the pull queue tasks. Note that this only happens if - * the mapper completes execution without errors. - */ - public static class RefreshDnsOnHostRenameReducer extends Reducer { - - private static final long serialVersionUID = 9077366205249562118L; - - @NonFinalForTesting - private static AsyncTaskMetrics asyncTaskMetrics = new AsyncTaskMetrics(new SystemClock()); - - private final Lock lock; - private final Retrier retrier; - private final List refreshRequests; - - RefreshDnsOnHostRenameReducer( - List refreshRequests, Lock lock, Retrier retrier) { - this.refreshRequests = refreshRequests; - this.lock = lock; - this.retrier = retrier; - } - - @Override - public void reduce(Boolean key, ReducerInput values) { - // The reduce() method is run precisely once, because the NullInput caused the mapper to emit - // a dummy value once. - deleteTasksWithRetry( - refreshRequests, - getQueue(QUEUE_ASYNC_HOST_RENAME), - asyncTaskMetrics, - retrier, - OperationResult.SUCCESS); - - lock.release(); - } - } - - /** Deletes a list of tasks from the given queue using a retrier. */ - private static void deleteTasksWithRetry( - final List refreshRequests, - final Queue queue, - AsyncTaskMetrics asyncTaskMetrics, - Retrier retrier, - OperationResult result) { - if (refreshRequests.isEmpty()) { - return; - } - final List tasks = - refreshRequests.stream().map(DnsRefreshRequest::task).collect(toImmutableList()); - retrier.callWithRetry(() -> queue.deleteTask(tasks), TransientFailureException.class); - refreshRequests.forEach( - r -> asyncTaskMetrics.recordAsyncFlowResult(DNS_REFRESH, result, r.requestedTime())); - } - - /** A class that encapsulates the values of a request to refresh DNS for a renamed host. */ - @AutoValue - abstract static class DnsRefreshRequest implements Serializable { - - private static final long serialVersionUID = 1772812852271288622L; - - abstract VKey hostKey(); - - abstract DateTime lastUpdateTime(); - abstract DateTime requestedTime(); - abstract boolean isRefreshNeeded(); - abstract TaskHandle task(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setHostKey(VKey hostKey); - - abstract Builder setLastUpdateTime(DateTime lastUpdateTime); - abstract Builder setRequestedTime(DateTime requestedTime); - abstract Builder setIsRefreshNeeded(boolean isRefreshNeeded); - abstract Builder setTask(TaskHandle task); - abstract DnsRefreshRequest build(); - } - - /** - * Returns a packaged-up {@link DnsRefreshRequest} parsed from a task queue task. - */ - static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception { - ImmutableMap params = ImmutableMap.copyOf(task.extractParams()); - VKey hostKey = - VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified")); - HostResource host = - tm().transact(() -> tm().loadByKeyIfPresent(hostKey)) - .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist")); - boolean isHostDeleted = - isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp())); - if (isHostDeleted) { - logger.atInfo().log("Host %s is already deleted, not refreshing DNS.", hostKey); - } - return new AutoValue_RefreshDnsOnHostRenameAction_DnsRefreshRequest.Builder() - .setHostKey(hostKey) - .setLastUpdateTime(host.getUpdateTimestamp().getTimestamp()) - .setRequestedTime( - DateTime.parse( - checkNotNull(params.get(PARAM_REQUESTED_TIME), "Requested time not specified"))) - .setIsRefreshNeeded(!isHostDeleted) - .setTask(task) - .build(); - } - } -} diff --git a/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java b/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java index d170bf543..03b3501a0 100644 --- a/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java +++ b/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java @@ -23,8 +23,6 @@ import com.google.common.collect.Streams; import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier; import google.registry.model.UpdateAutoTimestamp; import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.replay.SqlEntity; import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.persistence.transaction.TransactionManagerFactory; import java.io.Serializable; @@ -235,10 +233,6 @@ public final class RegistryJpaIO { @ProcessElement public void processElement(OutputReceiver outputReceiver) { - // Preload the migration schedule into cache, otherwise the cache loading query may happen - // before the setDatabaseSnapshot call in the transaction below, causing it to fail. - DatabaseMigrationStateSchedule.get(); - jpaTm() .transactNoRetry( () -> { @@ -433,11 +427,23 @@ public final class RegistryJpaIO { } } + /** Returns this entity's primary key field(s) in a string. */ private String toEntityKeyString(Object entity) { - if (entity instanceof SqlEntity) { - return ((SqlEntity) entity).getPrimaryKeyString(); + try { + return jpaTm() + .transact( + () -> + String.format( + "%s_%s", + entity.getClass().getSimpleName(), + jpaTm() + .getEntityManager() + .getEntityManagerFactory() + .getPersistenceUnitUtil() + .getIdentifier(entity))); + } catch (IllegalArgumentException e) { + return "Non-SqlEntity: " + entity; } - return "Non-SqlEntity: " + String.valueOf(entity); } } } diff --git a/core/src/main/java/google/registry/beam/initsql/BackupPaths.java b/core/src/main/java/google/registry/beam/initsql/BackupPaths.java deleted file mode 100644 index ea5b37f62..000000000 --- a/core/src/main/java/google/registry/beam/initsql/BackupPaths.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Strings.isNullOrEmpty; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Streams; -import google.registry.model.annotations.DeleteAfterMigration; -import org.joda.time.DateTime; - -/** - * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a - * Datastore export and Nomulus CommitLogs that overlap with the export. - */ -@DeleteAfterMigration -public final class BackupPaths { - - private BackupPaths() {} - - private static final String WILDCARD_CHAR = "*"; - private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/output-%s"; - - public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_"; - private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*"; - - /** - * Pattern of the per-project file with Cloud SQL connection information. To get a concrete path, - * user needs to provide the name of the environment, alpha, crash, sandbox, or production. This - * file is meant for applications without access to secrets stored in Datastore. - * - *

In production, this is an base-64 encoded encrypted file with one line, which contains - * space-separated values of Cloud SQL instance name, login, and password. - * - *

A plain text may be used for tests to a local database. Replace Cloud SQL instance name with - * JDBC URL. - */ - private static final String SQL_CONN_INFO_FILE_PATTERN = - "gs://domain-registry-dev-deploy/cloudsql-credentials/%s/admin_credential.enc"; - - private static final ImmutableSet ALLOWED_ENV = - ImmutableSet.of("alpha", "crash", "sandbox", "production"); - - /** - * Returns a regex pattern that matches all Datastore export files of a given {@code kind}. - * - * @param exportDir path to the top directory of a Datastore export - * @param kind the 'kind' of the Datastore entity - */ - public static String getExportFileNamePattern(String exportDir, String kind) { - checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir."); - checkArgument(!isNullOrEmpty(kind), "Null or empty kind."); - return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, WILDCARD_CHAR); - } - - /** - * Returns an {@link ImmutableList} of regex patterns that match all Datastore export files of the - * given {@code kinds}. - * - * @param exportDir path to the top directory of a Datastore export - * @param kinds all entity 'kinds' to be matched - */ - public static ImmutableList getExportFilePatterns( - String exportDir, Iterable kinds) { - checkNotNull(kinds, "kinds"); - return Streams.stream(kinds) - .map(kind -> getExportFileNamePattern(exportDir, kind)) - .collect(ImmutableList.toImmutableList()); - } - - /** - * Returns the fully qualified path of a Datastore export file with the given {@code kind} and - * {@code shard}. - * - * @param exportDir path to the top directory of a Datastore export - * @param kind the 'kind' of the Datastore entity - * @param shard an integer suffix of the file name - */ - public static String getExportFileNameByShard(String exportDir, String kind, int shard) { - checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir."); - checkArgument(!isNullOrEmpty(kind), "Null or empty kind."); - checkArgument(shard >= 0, "Negative shard %s not allowed.", shard); - return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, shard); - } - - /** Returns an {@link ImmutableList} of regex patterns that match all CommitLog files. */ - public static ImmutableList getCommitLogFilePatterns(String commitLogDir) { - return ImmutableList.of(String.format(COMMIT_LOG_PATTERN_TEMPLATE, commitLogDir)); - } - - /** Gets the Commit timestamp from a CommitLog file name. */ - public static DateTime getCommitLogTimestamp(String fileName) { - checkArgument(!isNullOrEmpty(fileName), "Null or empty fileName."); - int start = fileName.lastIndexOf(COMMIT_LOG_NAME_PREFIX); - checkArgument(start >= 0, "Illegal file name %s.", fileName); - return DateTime.parse(fileName.substring(start + COMMIT_LOG_NAME_PREFIX.length())); - } - - public static ImmutableList getCloudSQLCredentialFilePatterns(String environmentName) { - checkArgument( - ALLOWED_ENV.contains(environmentName), "Invalid environment name %s", environmentName); - return ImmutableList.of(String.format(SQL_CONN_INFO_FILE_PATTERN, environmentName)); - } -} diff --git a/core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java b/core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java deleted file mode 100644 index a646a3d50..000000000 --- a/core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.appengine.api.datastore.Entity; -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Objects; - -/** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */ -@DeleteAfterMigration -final class DomainBaseUtil { - - private DomainBaseUtil() {} - - /** - * Removes properties that contain foreign keys from a Datastore {@link Entity} that represents an - * Ofy {@link google.registry.model.domain.DomainBase}. This breaks the cycle of foreign key - * constraints between entity kinds, allowing {@code DomainBases} to be inserted into the SQL - * database. See {@link InitSqlPipeline} for a use case, where the full {@code DomainBases} are - * written again during the last stage of the pipeline. - * - *

The returned object may be in bad state. Specifically, {@link - * google.registry.model.eppcommon.StatusValue#INACTIVE} is not added after name servers are - * removed. This only impacts tests that manipulate Datastore entities directly. - * - *

This operation is performed on an Datastore {@link Entity} instead of Ofy Java object - * because Objectify requires access to a Datastore service when converting an Ofy object to a - * Datastore {@code Entity}. If we insist on working with Objectify objects, we face a few - * unsatisfactory options: - * - *

- * - *

Given our use case, operating on Datastore entities is the best option. - * - * @throws IllegalArgumentException if input does not represent a DomainBase - */ - static Entity removeBillingAndPollAndHosts(Entity domainBase) { - checkNotNull(domainBase, "domainBase"); - checkArgument( - Objects.equals(domainBase.getKind(), "DomainBase"), - "Expecting DomainBase, got %s", - domainBase.getKind()); - Entity clone = domainBase.clone(); - clone.removeProperty("autorenewBillingEvent"); - clone.removeProperty("autorenewPollMessage"); - clone.removeProperty("deletePollMessage"); - clone.removeProperty("nsHosts"); - domainBase.getProperties().keySet().stream() - .filter(s -> s.startsWith("transferData.")) - .forEach(s -> clone.removeProperty(s)); - domainBase.getProperties().keySet().stream() - .filter(s -> s.startsWith("gracePeriods.")) - .forEach(s -> clone.removeProperty(s)); - return clone; - } -} diff --git a/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java b/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java deleted file mode 100644 index 3f17dcc3e..000000000 --- a/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.backup.VersionedEntity; -import google.registry.beam.common.RegistryJpaIO; -import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.billing.BillingEvent; -import google.registry.model.common.Cursor; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.token.AllocationToken; -import google.registry.model.host.HostResource; -import google.registry.model.poll.PollMessage; -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.tld.Registry; -import google.registry.persistence.PersistenceModule.TransactionIsolationLevel; -import java.io.Serializable; -import java.util.Collection; -import java.util.Optional; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.transforms.Wait; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.joda.time.DateTime; - -/** - * A BEAM pipeline that populates a SQL database with data from a Datastore backup. - * - *

This pipeline migrates EPP resources and related entities that cross-reference each other. To - * avoid violating foreign key constraints, writes to SQL are ordered by entity kinds. In addition, - * the {@link DomainBase} kind is written twice (see details below). The write order is presented - * below. Although some kinds can be written concurrently, e.g. {@code ContactResource} and {@code - * RegistrarContact}, we do not expect any performance benefit since the limiting resource is the - * number of JDBC connections. Google internal users may refer to the design doc for more information. - * - *

    - *
  1. {@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set - * up in the SQL database. - *
  2. {@link Cursor}: Logically can depend on {@code Registry}, but without foreign key. - *
  3. {@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet. - *
  4. {@link ContactResource}: references {@code Registrar} - *
  5. {@link RegistrarContact}: references {@code Registrar}. - *
  6. Cleansed {@link DomainBase}: with references to {@code BillingEvent}, {@code Recurring}, - * {@code Cancellation} and {@code HostResource} removed, still references {@code Registrar} - * and {@code ContactResource}. The removal breaks circular Foreign Key references. - *
  7. {@link HostResource}: references {@code DomainBase}. - *
  8. {@link HistoryEntry}: maps to one of three SQL entity types and may reference {@code - * Registrar}, {@code ContactResource}, {@code HostResource}, and {@code DomainBase}. - *
  9. {@link AllocationToken}: references {@code HistoryEntry}. - *
  10. {@link BillingEvent.Recurring}: references {@code Registrar}, {@code DomainBase} and {@code - * HistoryEntry}. - *
  11. {@link BillingEvent.OneTime}: references {@code Registrar}, {@code DomainBase}, {@code - * BillingEvent.Recurring}, {@code HistoryEntry} and {@code AllocationToken}. - *
  12. {@link BillingEvent.Cancellation}: references {@code Registrar}, {@code DomainBase}, {@code - * BillingEvent.Recurring}, {@code BillingEvent.OneTime}, and {@code HistoryEntry}. - *
  13. {@link PollMessage}: references {@code Registrar}, {@code DomainBase}, {@code - * ContactResource}, {@code HostResource}, and {@code HistoryEntry}. - *
  14. {@link DomainBase}, original copy from Datastore. - *
- * - *

This pipeline expects that the source Datastore has at least one entity in each of the types - * above. This assumption allows us to construct a simpler pipeline graph that can be visually - * examined, and is true in all intended use cases. However, tests must not violate this assumption - * when setting up data, otherwise they may run into foreign key constraint violations. The reason - * is that this pipeline uses the {@link Wait} transform to order the persistence by entity type. - * However, the wait is skipped if the target type has no data, resulting in subsequent entity types - * starting prematurely. E.g., if a Datastore has no {@code RegistrarContact} entities, the pipeline - * may start writing {@code DomainBase} entities before all {@code Registry}, {@code Registrar} and - * {@code ContactResource} entities have been persisted. - */ -@DeleteAfterMigration -public class InitSqlPipeline implements Serializable { - - /** - * Datastore kinds to be written to the SQL database before the cleansed version of {@link - * DomainBase}. - */ - private static final ImmutableList> PHASE_ONE_ORDERED = - ImmutableList.of( - Registry.class, - Cursor.class, - Registrar.class, - ContactResource.class, - RegistrarContact.class); - - /** - * Datastore kinds to be written to the SQL database after the cleansed version of {@link - * DomainBase}. - */ - private static final ImmutableList> PHASE_TWO_ORDERED = - ImmutableList.of( - HostResource.class, - HistoryEntry.class, - AllocationToken.class, - BillingEvent.Recurring.class, - BillingEvent.OneTime.class, - BillingEvent.Cancellation.class, - PollMessage.class, - DomainBase.class); - - private final InitSqlPipelineOptions options; - - InitSqlPipeline(InitSqlPipelineOptions options) { - this.options = options; - } - - PipelineResult run() { - return run(Pipeline.create(options)); - } - - @VisibleForTesting - PipelineResult run(Pipeline pipeline) { - setupPipeline(pipeline); - return pipeline.run(); - } - - @VisibleForTesting - void setupPipeline(Pipeline pipeline) { - options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_UNCOMMITTED); - PCollectionTuple datastoreSnapshot = - pipeline.apply( - "Load Datastore snapshot", - Transforms.loadDatastoreSnapshot( - options.getDatastoreExportDir(), - options.getCommitLogDir(), - DateTime.parse(options.getCommitLogStartTimestamp()), - DateTime.parse(options.getCommitLogEndTimestamp()), - ImmutableSet.builder() - .add("DomainBase") - .addAll(toKindStrings(PHASE_ONE_ORDERED)) - .addAll(toKindStrings(PHASE_TWO_ORDERED)) - .build())); - - // Set up the pipeline to write entity kinds from PHASE_ONE_ORDERED to SQL. Return an object - // that signals the completion of the phase. - PCollection blocker = - scheduleOnePhaseWrites(datastoreSnapshot, PHASE_ONE_ORDERED, Optional.empty(), null); - blocker = - writeToSql( - "DomainBase without circular foreign keys", - removeDomainBaseForeignKeys(datastoreSnapshot) - .apply("Wait on phase one", Wait.on(blocker))); - // Set up the pipeline to write entity kinds from PHASE_TWO_ORDERED to SQL. This phase won't - // start until all cleansed DomainBases have been written (started by line above). - scheduleOnePhaseWrites( - datastoreSnapshot, PHASE_TWO_ORDERED, Optional.of(blocker), "DomainBaseNoFkeys"); - } - - private PCollection removeDomainBaseForeignKeys( - PCollectionTuple datastoreSnapshot) { - PCollection domainBases = - datastoreSnapshot.get(Transforms.createTagForKind("DomainBase")); - return domainBases.apply( - "Remove circular foreign keys from DomainBase", - ParDo.of(new RemoveDomainBaseForeignKeys())); - } - - /** - * Sets up the pipeline to write entities in {@code entityClasses} to SQL. Entities are written - * one kind at a time based on each kind's position in {@code entityClasses}. Concurrency exists - * within each kind. - * - * @param datastoreSnapshot the Datastore snapshot of all data to be migrated to SQL - * @param entityClasses the entity types in write order - * @param blockingPCollection the pipeline stage that blocks this phase - * @param blockingTag description of the stage (if exists) that blocks this phase. Needed for - * generating unique transform ids - * @return the output {@code PCollection} from the writing of the last entity kind. Other parts of - * the pipeline can {@link Wait} on this object - */ - private PCollection scheduleOnePhaseWrites( - PCollectionTuple datastoreSnapshot, - Collection> entityClasses, - Optional> blockingPCollection, - String blockingTag) { - checkArgument(!entityClasses.isEmpty(), "Each phase must have at least one kind."); - ImmutableList> tags = - toKindStrings(entityClasses).stream() - .map(Transforms::createTagForKind) - .collect(ImmutableList.toImmutableList()); - - PCollection prev = blockingPCollection.orElse(null); - String prevTag = blockingTag; - for (TupleTag tag : tags) { - PCollection curr = datastoreSnapshot.get(tag); - if (prev != null) { - curr = curr.apply("Wait on " + prevTag, Wait.on(prev)); - } - prev = writeToSql(tag.getId(), curr); - prevTag = tag.getId(); - } - return prev; - } - - private PCollection writeToSql(String transformId, PCollection data) { - return data.apply( - "Write to Sql: " + transformId, - RegistryJpaIO.write() - .withName(transformId) - .withBatchSize(options.getSqlWriteBatchSize()) - .withShards(options.getSqlWriteShards()) - .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity) - .disableUpdateAutoTimestamp()); - } - - private static ImmutableList toKindStrings(Collection> entityClasses) { - return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList()); - } - - public static void main(String[] args) { - InitSqlPipelineOptions options = - PipelineOptionsFactory.fromArgs(args).withValidation().as(InitSqlPipelineOptions.class); - new InitSqlPipeline(options).run(); - } -} diff --git a/core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java b/core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java deleted file mode 100644 index b4f3a1940..000000000 --- a/core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import google.registry.beam.common.RegistryPipelineOptions; -import google.registry.model.annotations.DeleteAfterMigration; -import org.apache.beam.sdk.options.Description; -import org.apache.beam.sdk.options.Validation; - -/** Pipeline options for {@link InitSqlPipeline} */ -@DeleteAfterMigration -public interface InitSqlPipelineOptions extends RegistryPipelineOptions { - - @Description("The root directory of the export to load.") - String getDatastoreExportDir(); - - void setDatastoreExportDir(String datastoreExportDir); - - @Description("The directory that contains all CommitLog files.") - String getCommitLogDir(); - - void setCommitLogDir(String commitLogDir); - - @Description("The earliest CommitLogs to load, in ISO8601 format.") - @Validation.Required - String getCommitLogStartTimestamp(); - - void setCommitLogStartTimestamp(String commitLogStartTimestamp); - - @Description("The latest CommitLogs to load, in ISO8601 format.") - @Validation.Required - String getCommitLogEndTimestamp(); - - void setCommitLogEndTimestamp(String commitLogEndTimestamp); -} diff --git a/core/src/main/java/google/registry/beam/initsql/README.md b/core/src/main/java/google/registry/beam/initsql/README.md deleted file mode 100644 index 19c54200e..000000000 --- a/core/src/main/java/google/registry/beam/initsql/README.md +++ /dev/null @@ -1,17 +0,0 @@ -## Summary - -This package contains a BEAM pipeline that populates a Cloud SQL database from a -Datastore backup. The pipeline uses an unsynchronized Datastore export and -overlapping CommitLogs generated by the Nomulus server to recreate a consistent -Datastore snapshot, and writes the data to a Cloud SQL instance. - -## Pipeline Visualization - -The golden flow graph of the InitSqlPipeline is saved both as a text-base -[DOT file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.dot) -and a -[.png file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.png). -A test compares the flow graph of the current pipeline with the golden graph, -and will fail if changes are detected. When this happens, run the Gradle task -':core:updateInitSqlPipelineGraph' to update the golden files and review the -changes. diff --git a/core/src/main/java/google/registry/beam/initsql/Transforms.java b/core/src/main/java/google/registry/beam/initsql/Transforms.java deleted file mode 100644 index fa969592b..000000000 --- a/core/src/main/java/google/registry/beam/initsql/Transforms.java +++ /dev/null @@ -1,485 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; -import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp; -import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static google.registry.util.DateTimeUtils.isBeforeOrAt; -import static google.registry.util.DomainNameUtils.canonicalizeHostname; -import static java.util.Comparator.comparing; -import static org.apache.beam.sdk.values.TypeDescriptors.kvs; -import static org.apache.beam.sdk.values.TypeDescriptors.strings; - -import avro.shaded.com.google.common.collect.Iterators; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityTranslator; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Streams; -import google.registry.backup.CommitLogImports; -import google.registry.backup.VersionedEntity; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.billing.BillingEvent.Flag; -import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.domain.DomainBase; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.SqlEntity; -import google.registry.model.reporting.HistoryEntry; -import google.registry.tools.LevelDbLogReader; -import java.io.Serializable; -import java.util.Collection; -import java.util.Iterator; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import javax.annotation.Nullable; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.Compression; -import org.apache.beam.sdk.io.FileIO; -import org.apache.beam.sdk.io.FileIO.ReadableFile; -import org.apache.beam.sdk.io.fs.EmptyMatchTreatment; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.Flatten; -import org.apache.beam.sdk.transforms.GroupByKey; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.transforms.ProcessFunction; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.joda.time.DateTime; - -/** - * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export - * files and Nomulus CommitLog files. - */ -@DeleteAfterMigration -public final class Transforms { - - private Transforms() {} - - /** - * The commitTimestamp assigned to all entities loaded from a Datastore export file. The exact - * value does not matter, but it must be lower than the timestamps of real CommitLog records. - */ - @VisibleForTesting static final long EXPORT_ENTITY_TIME_STAMP = START_OF_TIME.getMillis(); - - /** - * Returns a {@link TupleTag} that can be used to retrieve entities of the given {@code kind} from - * the Datastore snapshot returned by {@link #loadDatastoreSnapshot}. - */ - public static TupleTag createTagForKind(String kind) { - // When used with PCollectionTuple the result must retain generic type information. - // Both the Generic param and the empty bracket below are important. - return new TupleTag(Transforms.class.getSimpleName() + ":" + kind) {}; - } - - /** - * Composite {@link PTransform transform} that loads the Datastore snapshot right before {@code - * commitLogToTime} for caller specified {@code kinds}. The resulting snapshot has all changes - * that happened before {@code commitLogToTime}, and none at or after {@code commitLogToTime}. - * - *

Caller must provide the location of a Datastore export that started AFTER {@code - * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory - * of all CommitLog files. - * - *

Selection of {@code commitLogFromTime} and {@code commitLogToTime} should follow the - * guidelines below to ensure that all incremental changes concurrent with the export are covered: - * - *

- * - *

The output from the returned transform is a {@link PCollectionTuple} consisting of {@link - * VersionedEntity VersionedEntities} grouped into {@link PCollection PCollections} by {@code - * kind}. - */ - public static PTransform loadDatastoreSnapshot( - String exportDir, - String commitLogDir, - DateTime commitLogFromTime, - DateTime commitLogToTime, - Set kinds) { - checkArgument(kinds != null && !kinds.isEmpty(), "At least one kind is expected."); - - // Create tags to collect entities by kind in final step. - final ImmutableMap> outputTags = - kinds.stream() - .collect(ImmutableMap.toImmutableMap(kind -> kind, Transforms::createTagForKind)); - // Arbitrarily select one tag as mainOutTag and put the remaining ones in a TupleTagList. - // This separation is required by ParDo's config API. - Iterator> tagsIt = outputTags.values().iterator(); - final TupleTag mainOutputTag = tagsIt.next(); - final TupleTagList additionalTags = TupleTagList.of(ImmutableList.copyOf(tagsIt)); - - return new PTransform() { - @Override - public PCollectionTuple expand(PBegin input) { - PCollection exportedEntities = - input - .apply("Get export file patterns", getDatastoreExportFilePatterns(exportDir, kinds)) - .apply("Find export files", getFilesByPatterns()) - .apply("Load export data", loadExportDataFromFiles()); - PCollection commitLogEntities = - input - .apply("Get commitlog file patterns", getCommitLogFilePatterns(commitLogDir)) - .apply("Find commitlog files", getFilesByPatterns()) - .apply( - "Filter commitLog by time", - filterCommitLogsByTime(commitLogFromTime, commitLogToTime)) - .apply("Load commitlog data", loadCommitLogsFromFiles(kinds)); - return PCollectionList.of(exportedEntities) - .and(commitLogEntities) - .apply("Merge exports and CommitLogs", Flatten.pCollections()) - .apply( - "Key entities by Datastore Keys", - // Converting to KV instead of KV b/c default coder for Key - // (SerializableCoder) is not deterministic and cannot be used with GroupBy. - MapElements.into(kvs(strings(), TypeDescriptor.of(VersionedEntity.class))) - .via((VersionedEntity e) -> KV.of(e.key().toString(), e))) - .apply("Gather entities by key", GroupByKey.create()) - .apply( - "Output latest version per entity", - ParDo.of( - new DoFn>, VersionedEntity>() { - @ProcessElement - public void processElement( - @Element KV> kv, - MultiOutputReceiver out) { - Optional latest = - Streams.stream(kv.getValue()) - .sorted(comparing(VersionedEntity::commitTimeMills).reversed()) - .findFirst(); - // Throw to abort (after default retries). Investigate, fix, and rerun. - checkState( - latest.isPresent(), "Unexpected key with no data", kv.getKey()); - if (latest.get().isDelete()) { - return; - } - String kind = latest.get().getEntity().get().getKind(); - out.get(outputTags.get(kind)).output(latest.get()); - } - }) - .withOutputTags(mainOutputTag, additionalTags)); - } - }; - } - - /** - * Returns a {@link PTransform transform} that can generate a collection of patterns that match - * all Datastore CommitLog files. - */ - public static PTransform> getCommitLogFilePatterns( - String commitLogDir) { - return toStringPCollection(BackupPaths.getCommitLogFilePatterns(commitLogDir)); - } - - /** - * Returns a {@link PTransform transform} that can generate a collection of patterns that match - * all Datastore export files of the given {@code kinds}. - */ - public static PTransform> getDatastoreExportFilePatterns( - String exportDir, Collection kinds) { - return toStringPCollection(getExportFilePatterns(exportDir, kinds)); - } - - public static PTransform> getCloudSqlConnectionInfoFilePatterns( - String gcpProjectName) { - return toStringPCollection(BackupPaths.getCloudSQLCredentialFilePatterns(gcpProjectName)); - } - - /** - * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}. - */ - public static PTransform, PCollection> getFilesByPatterns() { - return new PTransform, PCollection>() { - @Override - public PCollection expand(PCollection input) { - return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW)); - } - }; - } - - /** - * Returns CommitLog files with timestamps between {@code fromTime} (inclusive) and {@code - * endTime} (exclusive). - */ - public static PTransform, PCollection> - filterCommitLogsByTime(DateTime fromTime, DateTime toTime) { - return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime)); - } - - /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */ - public static PTransform, PCollection> - loadExportDataFromFiles() { - return processFiles( - new BackupFileReader( - file -> - Iterators.transform( - LevelDbLogReader.from(file.open()), - (byte[] bytes) -> VersionedEntity.from(EXPORT_ENTITY_TIME_STAMP, bytes)))); - } - - /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */ - public static PTransform, PCollection> - loadCommitLogsFromFiles(Set kinds) { - return processFiles( - new BackupFileReader( - file -> - CommitLogImports.loadEntities(file.open()).stream() - .filter(e -> kinds.contains(e.key().getKind())) - .iterator())); - } - - // Production data repair configs go below. See b/185954992. Note that the CommitLog replay - // process does not filter out the ignored entities listed below, a mistake that we do not fix - // for operational convenience. Instead, the Database comparison tool will filter them out. See - // ValidateSqlUtils.java for more information. - - // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized - // history. They can be safely ignored. - public static final ImmutableSet IGNORED_DOMAINS = - ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT"); - - // Prober hosts referencing phantom registrars. They and their associated history entries can be - // safely ignored. - public static final ImmutableSet IGNORED_HOSTS = - ImmutableSet.of( - "4E21_WJ0TEST-GOOGLE", - "4E21_WJ1TEST-GOOGLE", - "4E21_WJ2TEST-GOOGLE", - "4E21_WJ3TEST-GOOGLE"); - - // Prober contacts referencing phantom registrars. They and their associated history entries can - // be safely ignored. - public static final ImmutableSet IGNORED_CONTACTS = - ImmutableSet.of( - "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE"); - - private static boolean isMigratable(Entity entity) { - // Checks specific to production data. See b/185954992 for details. - // The names of these bad entities in production do not conflict with other environments. For - // simplicities sake we apply them regardless of the source of the data. - if (entity.getKind().equals("DomainBase") - && IGNORED_DOMAINS.contains(entity.getKey().getName())) { - return false; - } - if (entity.getKind().equals("ContactResource")) { - String roid = entity.getKey().getName(); - return !IGNORED_CONTACTS.contains(roid); - } - if (entity.getKind().equals("HostResource")) { - String roid = entity.getKey().getName(); - return !IGNORED_HOSTS.contains(roid); - } - if (entity.getKind().equals("HistoryEntry")) { - // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated. - // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot - // be deserialized. - if (Objects.equals(entity.getProperty("type"), "DOMAIN_APPLICATION_CREATE")) { - return false; - } - - // Remove production bad data: Histories of ignored EPP resources: - com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent(); - if (parentKey.getKind().equals("ContactResource")) { - String contactRoid = parentKey.getName(); - return !IGNORED_CONTACTS.contains(contactRoid); - } - if (parentKey.getKind().equals("HostResource")) { - String hostRoid = parentKey.getName(); - return !IGNORED_HOSTS.contains(hostRoid); - } - if (parentKey.getKind().equals("DomainBase")) { - String domainRoid = parentKey.getName(); - return !IGNORED_DOMAINS.contains(domainRoid); - } - } - return true; - } - - @VisibleForTesting - static Entity repairBadData(Entity entity) { - if (entity.getKind().equals("Cancellation") - && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) { - // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the - // AUTO_RENEW flag. Note: all affected entities have empty flags so we can simply assign - // instead of append. See b/185954992. - entity.setUnindexedProperty("reason", Reason.RENEW.name()); - entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name())); - } else if (entity.getKind().equals("DomainBase")) { - // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this. - entity.setIndexedProperty( - "fullyQualifiedDomainName", - canonicalizeHostname((String) entity.getProperty("fullyQualifiedDomainName"))); - } else if (entity.getKind().equals("HostResource")) { - entity.setIndexedProperty( - "fullyQualifiedHostName", - canonicalizeHostname((String) entity.getProperty("fullyQualifiedHostName"))); - } - return entity; - } - - private static SqlEntity toSqlEntity(Object ofyEntity) { - if (ofyEntity instanceof HistoryEntry) { - HistoryEntry ofyHistory = (HistoryEntry) ofyEntity; - return (SqlEntity) ofyHistory.toChildHistoryEntity(); - } - return ((DatastoreAndSqlEntity) ofyEntity).toSqlEntity().get(); - } - - /** - * Converts a {@link VersionedEntity} to an JPA entity for persistence. - * - * @return An object to be persisted to SQL, or null if the input is not to be migrated. (Not - * using Optional in return because as a one-use method, we do not want to invest the effort - * to make Optional work with BEAM) - */ - @Nullable - public static SqlEntity convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) { - return dsEntity - .getEntity() - .filter(Transforms::isMigratable) - .map(Transforms::repairBadData) - .map(e -> auditedOfy().toPojo(e)) - .map(Transforms::toSqlEntity) - .orElse(null); - } - - /** Interface for serializable {@link Supplier suppliers}. */ - public interface SerializableSupplier extends Supplier, Serializable {} - - /** - * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the - * given {@link Iterable}. - */ - private static PTransform> toStringPCollection( - Iterable strings) { - return Create.of(strings).withCoder(StringUtf8Coder.of()); - } - - /** - * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using - * caller-provided {@code transformer}. - */ - private static PTransform, PCollection> processFiles( - DoFn transformer) { - return new PTransform, PCollection>() { - @Override - public PCollection expand(PCollection input) { - return input - .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED)) - .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer)); - } - }; - } - - private static class FilterCommitLogFileByTime extends DoFn { - private final DateTime fromTime; - private final DateTime toTime; - - FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) { - checkNotNull(fromTime, "fromTime"); - checkNotNull(toTime, "toTime"); - checkArgument( - fromTime.isBefore(toTime), - "Invalid time range: fromTime (%s) is before endTime (%s)", - fromTime, - toTime); - this.fromTime = fromTime; - this.toTime = toTime; - } - - @ProcessElement - public void processElement(@Element Metadata fileMeta, OutputReceiver out) { - DateTime timestamp = getCommitLogTimestamp(fileMeta.resourceId().toString()); - if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) { - out.output(fileMeta); - } - } - } - - /** - * Reads from a Datastore backup file and converts its content into {@link VersionedEntity - * VersionedEntities}. - * - *

The input file may be either a LevelDb file from a Datastore export or a CommitLog file - * generated by the Nomulus server. In either case, the file contains variable-length records and - * must be read sequentially from the beginning. If the read fails, the file needs to be retried - * from the beginning. - */ - private static class BackupFileReader extends DoFn { - private final ProcessFunction> reader; - - private BackupFileReader(ProcessFunction> reader) { - this.reader = reader; - } - - @ProcessElement - public void processElement(@Element ReadableFile file, OutputReceiver out) { - try { - reader.apply(file).forEachRemaining(out::output); - } catch (Exception e) { - // Let the pipeline use default retry strategy on the whole file. For GCP Dataflow this - // means retrying up to 4 times (may include other files grouped with this one), and failing - // the pipeline if no success. - throw new RuntimeException(e); - } - } - } - - /** - * Removes BillingEvents, {@link google.registry.model.poll.PollMessage PollMessages} and {@link - * google.registry.model.host.HostResource} from a {@link DomainBase}. These are circular foreign - * key constraints that prevent migration of {@code DomainBase} to SQL databases. - * - *

See {@link InitSqlPipeline} for more information. - */ - static class RemoveDomainBaseForeignKeys extends DoFn { - - @ProcessElement - public void processElement( - @Element VersionedEntity domainBase, OutputReceiver out) { - checkArgument( - domainBase.getEntity().isPresent(), "Unexpected delete entity %s", domainBase.key()); - Entity outputEntity = - DomainBaseUtil.removeBillingAndPollAndHosts(domainBase.getEntity().get()); - out.output( - VersionedEntity.from( - domainBase.commitTimeMills(), - EntityTranslator.convertToPb(outputEntity).toByteArray())); - } - } -} diff --git a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java index 44ad0a341..bb368b547 100644 --- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java +++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java @@ -41,11 +41,9 @@ import java.util.Optional; import java.util.regex.Pattern; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.coders.SerializableCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.io.TextIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.Contextful; import org.apache.beam.sdk.transforms.Count; @@ -92,28 +90,11 @@ public class InvoicingPipeline implements Serializable { void setupPipeline(Pipeline pipeline) { options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED); - PCollection billingEvents = - options.getDatabase().equals("DATASTORE") - ? readFromBigQuery(options, pipeline) - : readFromCloudSql(options, pipeline); - + PCollection billingEvents = readFromCloudSql(options, pipeline); saveInvoiceCsv(billingEvents, options); - saveDetailedCsv(billingEvents, options); } - static PCollection readFromBigQuery( - InvoicingPipelineOptions options, Pipeline pipeline) { - return pipeline.apply( - "Read BillingEvents from Bigquery", - BigQueryIO.read(BillingEvent::parseFromRecord) - .fromQuery(makeQuery(options.getYearMonth(), options.getProject())) - .withCoder(SerializableCoder.of(BillingEvent.class)) - .usingStandardSql() - .withoutValidation() - .withTemplateCompatibility()); - } - static PCollection readFromCloudSql( InvoicingPipelineOptions options, Pipeline pipeline) { Read read = diff --git a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java index 9a9fab709..ff4d5a69d 100644 --- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java +++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java @@ -30,11 +30,6 @@ public interface InvoicingPipelineOptions extends RegistryPipelineOptions { void setInvoiceFilePrefix(String value); - @Description("The database to read data from.") - String getDatabase(); - - void setDatabase(String value); - @Description("The GCS bucket URL for invoices and detailed reports to be uploaded.") String getBillingBucketUrl(); diff --git a/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java b/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java index 4107ea539..cdfa69e35 100644 --- a/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java +++ b/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java @@ -14,30 +14,19 @@ package google.registry.beam.spec11; -import static google.registry.beam.BeamUtils.checkFieldsNotNull; -import static google.registry.beam.BeamUtils.extractField; - import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; import java.io.Serializable; -import org.apache.avro.generic.GenericRecord; -import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; /** * A POJO representing a domain name and associated info, parsed from a {@code SchemaAndRecord}. * - *

This is a trivially serializable class that allows Beam to transform the results of a Bigquery - * query into a standard Java representation, giving us the type guarantees and ease of manipulation - * Bigquery lacks, while localizing any Bigquery-side failures to the {@link #parseFromRecord} - * function. + *

This is a trivially serializable class that allows Beam to transform the results of a SQL + * query into a standard Java representation. */ @AutoValue public abstract class DomainNameInfo implements Serializable { - private static final ImmutableList FIELD_NAMES = - ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress"); - /** Returns the fully qualified domain name. */ abstract String domainName(); @@ -50,28 +39,8 @@ public abstract class DomainNameInfo implements Serializable { /** Returns the email address of the registrar associated with this domain. */ abstract String registrarEmailAddress(); - /** - * Constructs a {@link DomainNameInfo} from an Apache Avro {@code SchemaAndRecord}. - * - * @see - * Apache AVRO GenericRecord - */ - static DomainNameInfo parseFromRecord(SchemaAndRecord schemaAndRecord) { - checkFieldsNotNull(FIELD_NAMES, schemaAndRecord); - GenericRecord record = schemaAndRecord.getRecord(); - return create( - extractField(record, "domainName"), - extractField(record, "domainRepoId"), - extractField(record, "registrarId"), - extractField(record, "registrarEmailAddress")); - } - /** * Creates a concrete {@link DomainNameInfo}. - * - *

This should only be used outside this class for testing- instances of {@link DomainNameInfo} - * should otherwise come from {@link #parseFromRecord}. */ @VisibleForTesting static DomainNameInfo create( diff --git a/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java b/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java index c11b55cfe..437de19ed 100644 --- a/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java +++ b/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java @@ -15,7 +15,6 @@ package google.registry.beam.spec11; import static com.google.common.base.Preconditions.checkArgument; -import static google.registry.beam.BeamUtils.getQueryFromFile; import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import com.google.auto.value.AutoValue; @@ -33,15 +32,12 @@ import google.registry.model.reporting.Spec11ThreatMatch.ThreatType; import google.registry.persistence.PersistenceModule.TransactionIsolationLevel; import google.registry.persistence.VKey; import google.registry.util.Retrier; -import google.registry.util.SqlTemplate; import google.registry.util.UtilsModule; import java.io.Serializable; import javax.inject.Singleton; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.coders.SerializableCoder; import org.apache.beam.sdk.io.TextIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.GroupByKey; @@ -104,10 +100,7 @@ public class Spec11Pipeline implements Serializable { void setupPipeline(Pipeline pipeline) { options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED); - PCollection domains = - options.getDatabase().equals("DATASTORE") - ? readFromBigQuery(options, pipeline) - : readFromCloudSql(pipeline); + PCollection domains = readFromCloudSql(pipeline); PCollection> threatMatches = domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn)); @@ -156,24 +149,6 @@ public class Spec11Pipeline implements Serializable { })); } - static PCollection readFromBigQuery( - Spec11PipelineOptions options, Pipeline pipeline) { - return pipeline.apply( - "Read active domains from BigQuery", - BigQueryIO.read(DomainNameInfo::parseFromRecord) - .fromQuery( - SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "domain_name_infos.sql")) - .put("PROJECT_ID", options.getProject()) - .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export") - .put("REGISTRAR_TABLE", "Registrar") - .put("DOMAIN_BASE_TABLE", "DomainBase") - .build()) - .withCoder(SerializableCoder.of(DomainNameInfo.class)) - .usingStandardSql() - .withoutValidation() - .withTemplateCompatibility()); - } - private static KV parseRow(Object[] row) { return KV.of((String) row[0], (String) row[1]); } diff --git a/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java b/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java index a04730b7c..7e3ab546b 100644 --- a/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java +++ b/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java @@ -34,9 +34,4 @@ public interface Spec11PipelineOptions extends RegistryPipelineOptions { String getReportingBucketUrl(); void setReportingBucketUrl(String value); - - @Description("The database to read data from.") - String getDatabase(); - - void setDatabase(String value); } diff --git a/core/src/main/java/google/registry/flows/EppException.java b/core/src/main/java/google/registry/flows/EppException.java index c4dd987f8..5dbef1faa 100644 --- a/core/src/main/java/google/registry/flows/EppException.java +++ b/core/src/main/java/google/registry/flows/EppException.java @@ -25,7 +25,6 @@ import google.registry.model.eppinput.EppInput.InnerCommand; import google.registry.model.eppinput.EppInput.ResourceCommandWrapper; import google.registry.model.eppoutput.Result; import google.registry.model.eppoutput.Result.Code; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import java.lang.annotation.Documented; import java.lang.annotation.Inherited; import java.lang.annotation.Retention; @@ -262,12 +261,4 @@ public abstract class EppException extends Exception { super("Specified protocol version is not implemented"); } } - - /** Registry is currently undergoing maintenance and is in read-only mode. */ - @EppResultCode(Code.COMMAND_FAILED) - public static class ReadOnlyModeEppException extends EppException { - ReadOnlyModeEppException(ReadOnlyModeException cause) { - super("Registry is currently undergoing maintenance and is in read-only mode", cause); - } - } } diff --git a/core/src/main/java/google/registry/flows/FlowRunner.java b/core/src/main/java/google/registry/flows/FlowRunner.java index d7747fc98..e1859ba7c 100644 --- a/core/src/main/java/google/registry/flows/FlowRunner.java +++ b/core/src/main/java/google/registry/flows/FlowRunner.java @@ -19,7 +19,6 @@ import static google.registry.xml.XmlTransformer.prettyPrint; import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; -import google.registry.flows.EppException.ReadOnlyModeEppException; import google.registry.flows.FlowModule.DryRun; import google.registry.flows.FlowModule.InputXml; import google.registry.flows.FlowModule.RegistrarId; @@ -29,7 +28,6 @@ import google.registry.flows.session.LoginFlow; import google.registry.model.eppcommon.Trid; import google.registry.model.eppoutput.EppOutput; import google.registry.monitoring.whitebox.EppMetric; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import javax.inject.Inject; import javax.inject.Provider; @@ -99,8 +97,6 @@ public class FlowRunner { return e.output; } catch (EppRuntimeException e) { throw e.getCause(); - } catch (ReadOnlyModeException e) { - throw new ReadOnlyModeEppException(e); } } diff --git a/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java b/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java index 8a6faf9a1..00c1230dd 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java @@ -50,7 +50,6 @@ import org.joda.time.DateTime; /** * An EPP flow that creates a new contact. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link ResourceAlreadyExistsForThisClientException} * @error {@link ResourceCreateContentionException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java b/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java index f8f636277..6e5a53627 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java @@ -24,7 +24,6 @@ import static google.registry.model.ResourceTransferUtils.denyPendingTransfer; import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete; import static google.registry.model.eppoutput.Result.Code.SUCCESS; import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import com.google.common.collect.ImmutableSet; @@ -58,7 +57,6 @@ import org.joda.time.DateTime; * references to the host before the deletion is allowed to proceed. A poll message will be written * with the success or failure message when the process is complete. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException} @@ -92,7 +90,6 @@ public final class ContactDeleteFlow implements TransactionalFlow { extensionManager.register(MetadataExtension.class); validateRegistrarIsLoggedIn(registrarId); extensionManager.validate(); - assertAsyncActionsAreAllowed(); DateTime now = tm().getTransactionTime(); checkLinkedDomains(targetId, now, ContactResource.class); ContactResource existingContact = loadAndVerifyExistence(ContactResource.class, targetId, now); diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java index 2a78ff562..a7bd4ca9f 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java @@ -54,7 +54,6 @@ import org.joda.time.DateTime; * transfer is automatically approved. Within that window, this flow allows the losing client to * explicitly approve the transfer request, which then becomes effective immediately. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java index dfc43776b..80f2b9678 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java @@ -54,7 +54,6 @@ import org.joda.time.DateTime; * transfer is automatically approved. Within that window, this flow allows the gaining client to * withdraw the transfer request. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java index 06287eeb2..63b9a7d3a 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java @@ -53,7 +53,6 @@ import org.joda.time.DateTime; * transfer is automatically approved. Within that window, this flow allows the losing client to * reject the transfer request. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java index fd98af00e..165ef00f8 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java @@ -63,7 +63,6 @@ import org.joda.time.Duration; * by the losing registrar or rejected, and the gaining registrar can also cancel the transfer * request. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java b/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java index 1f2d08cc7..ebbd81b2c 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java @@ -55,7 +55,6 @@ import org.joda.time.DateTime; /** * An EPP flow that updates a contact. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java b/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java index b5392f133..bc5baa6ff 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java @@ -139,7 +139,6 @@ import org.joda.time.Duration; * google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException} * @error {@link * google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException} - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException} * @error {@link ResourceAlreadyExistsForThisClientException} * @error {@link ResourceCreateContentionException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java b/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java index 60b673e7f..90ed74a0e 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java @@ -103,7 +103,6 @@ import org.joda.time.Duration; /** * An EPP flow that deletes a domain. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.EppException.UnimplementedExtensionException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java b/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java index 09fa4e8fc..d5ea48f12 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java @@ -95,7 +95,6 @@ import org.joda.time.Duration; * longer than 10 years unless it comes in at the exact millisecond that the domain would have * expired. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java b/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java index fac35b7c4..17aaba76c 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java @@ -94,7 +94,6 @@ import org.joda.time.DateTime; * restored to a single year expiration starting at the restore time, regardless of what the * original expiration time was. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.EppException.UnimplementedExtensionException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java index 9199fac2f..d8cedfb78 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java @@ -78,7 +78,6 @@ import org.joda.time.DateTime; * timestamps such that they only would become active when the transfer period passed. In this flow, * those speculative objects are deleted and replaced with new ones with the correct approval time. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java index 19d430ea9..102583129 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java @@ -65,7 +65,6 @@ import org.joda.time.DateTime; * timestamps such that they only would become active when the transfer period passed. In this flow, * those speculative objects are deleted. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java index ed39bf274..9055bbdc2 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java @@ -67,7 +67,6 @@ import org.joda.time.DateTime; * timestamps such that they only would become active when the transfer period passed. In this flow, * those speculative objects are deleted. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java index 913e32515..970fe90dd 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java @@ -94,7 +94,6 @@ import org.joda.time.DateTime; * rejection or cancellation of the request, they will be deleted (and in the approval case, * replaced with new ones with the correct approval time). * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java b/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java index c7a1bc850..2de0c3aaa 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java @@ -101,7 +101,6 @@ import org.joda.time.DateTime; * superuser. As such, adding or removing these statuses incurs a billing event. There will be only * one charge per update, even if several such statuses are updated at once. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.EppException.UnimplementedExtensionException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException} diff --git a/core/src/main/java/google/registry/flows/host/HostCreateFlow.java b/core/src/main/java/google/registry/flows/host/HostCreateFlow.java index 560c33e54..22bac414d 100644 --- a/core/src/main/java/google/registry/flows/host/HostCreateFlow.java +++ b/core/src/main/java/google/registry/flows/host/HostCreateFlow.java @@ -65,7 +65,6 @@ import org.joda.time.DateTime; * hosts cannot have any. This flow allows creating a host name, and if necessary enqueues tasks to * update DNS. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.IpAddressVersionMismatchException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link ResourceAlreadyExistsForThisClientException} diff --git a/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java b/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java index babdc068b..2299fb6d6 100644 --- a/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java +++ b/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java @@ -21,7 +21,6 @@ import static google.registry.flows.ResourceFlowUtils.verifyNoDisallowedStatuses import static google.registry.flows.ResourceFlowUtils.verifyResourceOwnership; import static google.registry.flows.host.HostFlowUtils.validateHostName; import static google.registry.model.eppoutput.Result.Code.SUCCESS; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import com.google.common.collect.ImmutableSet; @@ -55,7 +54,6 @@ import org.joda.time.DateTime; * references to the host before the deletion is allowed to proceed. A poll message will be written * with the success or failure message when the process is complete. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException} @@ -93,7 +91,6 @@ public final class HostDeleteFlow implements TransactionalFlow { extensionManager.register(MetadataExtension.class); validateRegistrarIsLoggedIn(registrarId); extensionManager.validate(); - assertAsyncActionsAreAllowed(); DateTime now = tm().getTransactionTime(); validateHostName(targetId); checkLinkedDomains(targetId, now, HostResource.class); diff --git a/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java b/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java index a396b564d..5921cecc8 100644 --- a/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java +++ b/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java @@ -28,7 +28,6 @@ import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomain import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomainOwnership; import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey; import static google.registry.model.reporting.HistoryEntry.Type.HOST_UPDATE; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.util.CollectionUtils.isNullOrEmpty; @@ -79,7 +78,6 @@ import org.joda.time.DateTime; * when it is renamed from external to internal at least one must be added. If the host is renamed * or IP addresses are added, tasks are enqueued to update DNS accordingly. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} @@ -137,9 +135,6 @@ public final class HostUpdateFlow implements TransactionalFlow { validateHostName(targetId); HostResource existingHost = loadAndVerifyExistence(HostResource.class, targetId, now); boolean isHostRename = suppliedNewHostName != null; - if (isHostRename) { - assertAsyncActionsAreAllowed(); - } String oldHostName = targetId; String newHostName = firstNonNull(suppliedNewHostName, oldHostName); DomainBase oldSuperordinateDomain = diff --git a/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java b/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java index 37ec23713..dab737f53 100644 --- a/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java +++ b/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; import google.registry.model.EppResource; import google.registry.model.ImmutableObject; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.index.EppResourceIndex; /** @@ -30,6 +31,7 @@ import google.registry.model.index.EppResourceIndex; *

The inputs provided by this class are not deletion-aware and do not project the resources * forward in time. That is the responsibility of mappers that use these inputs. */ +@DeleteAfterMigration public final class EppResourceInputs { private EppResourceInputs() {} diff --git a/core/src/main/java/google/registry/model/billing/BillingEvent.java b/core/src/main/java/google/registry/model/billing/BillingEvent.java index 5906a9dcb..e634d1359 100644 --- a/core/src/main/java/google/registry/model/billing/BillingEvent.java +++ b/core/src/main/java/google/registry/model/billing/BillingEvent.java @@ -46,8 +46,6 @@ import google.registry.model.domain.DomainHistory; import google.registry.model.domain.GracePeriod; import google.registry.model.domain.rgp.GracePeriodStatus; import google.registry.model.domain.token.AllocationToken; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.model.transfer.TransferData.TransferServerApproveEntity; import google.registry.persistence.BillingVKey.BillingEventVKey; import google.registry.persistence.BillingVKey.BillingRecurrenceVKey; @@ -347,7 +345,7 @@ public abstract class BillingEvent extends ImmutableObject }) @AttributeOverride(name = "id", column = @Column(name = "billing_event_id")) @WithLongVKey(compositeKey = true) - public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity { + public static class OneTime extends BillingEvent { /** The billable value. */ @Type(type = JodaMoneyType.TYPE_NAME) @@ -559,7 +557,7 @@ public abstract class BillingEvent extends ImmutableObject }) @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id")) @WithLongVKey(compositeKey = true) - public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity { + public static class Recurring extends BillingEvent { /** * The billing event recurs every year between {@link #eventTime} and this time on the @@ -696,7 +694,7 @@ public abstract class BillingEvent extends ImmutableObject }) @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id")) @WithLongVKey(compositeKey = true) - public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity { + public static class Cancellation extends BillingEvent { /** The billing time of the charge that is being cancelled. */ @Index @@ -819,7 +817,7 @@ public abstract class BillingEvent extends ImmutableObject @ReportedOn @Entity @WithLongVKey(compositeKey = true) - public static class Modification extends BillingEvent implements DatastoreOnlyEntity { + public static class Modification extends BillingEvent { /** The change in cost that should be applied to the original billing event. */ Money cost; diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java b/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java index 75ef8b14b..7fc845ab2 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java @@ -16,7 +16,6 @@ package google.registry.model.bulkquery; import google.registry.model.domain.DomainBase; import google.registry.model.domain.DomainContent; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import javax.persistence.Access; @@ -34,7 +33,7 @@ import javax.persistence.Entity; @Entity(name = "Domain") @WithStringVKey @Access(AccessType.FIELD) -public class DomainBaseLite extends DomainContent implements SqlOnlyEntity { +public class DomainBaseLite extends DomainContent { @Override @javax.persistence.Id diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java index 796d104a7..f50f41630 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java @@ -17,7 +17,6 @@ package google.registry.model.bulkquery; import com.google.common.base.Objects; import google.registry.model.domain.DomainHistory.DomainHistoryId; import google.registry.model.host.HostResource; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.VKey; import java.io.Serializable; import javax.persistence.Access; @@ -33,7 +32,7 @@ import javax.persistence.IdClass; @Entity @Access(AccessType.FIELD) @IdClass(DomainHistoryHost.class) -public class DomainHistoryHost implements Serializable, SqlOnlyEntity { +public class DomainHistoryHost implements Serializable { @Id private Long domainHistoryHistoryRevisionId; @Id private String domainHistoryDomainRepoId; diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java index 4891031b7..11af0cde5 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java @@ -20,7 +20,6 @@ import google.registry.model.domain.DomainContent; import google.registry.model.domain.DomainHistory; import google.registry.model.domain.DomainHistory.DomainHistoryId; import google.registry.model.domain.Period; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import javax.annotation.Nullable; @@ -48,7 +47,7 @@ import javax.persistence.PostLoad; @Entity(name = "DomainHistory") @Access(AccessType.FIELD) @IdClass(DomainHistoryId.class) -public class DomainHistoryLite extends HistoryEntry implements SqlOnlyEntity { +public class DomainHistoryLite extends HistoryEntry { // Store DomainContent instead of DomainBase so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainHost.java b/core/src/main/java/google/registry/model/bulkquery/DomainHost.java index 8055b5f4c..df427da90 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainHost.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainHost.java @@ -16,7 +16,6 @@ package google.registry.model.bulkquery; import com.google.common.base.Objects; import google.registry.model.host.HostResource; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.VKey; import java.io.Serializable; import javax.persistence.Access; @@ -29,7 +28,7 @@ import javax.persistence.IdClass; @Entity @Access(AccessType.FIELD) @IdClass(DomainHost.class) -public class DomainHost implements Serializable, SqlOnlyEntity { +public class DomainHost implements Serializable { @Id private String domainRepoId; diff --git a/core/src/main/java/google/registry/model/common/Cursor.java b/core/src/main/java/google/registry/model/common/Cursor.java index 2aca2ad80..7f28adc54 100644 --- a/core/src/main/java/google/registry/model/common/Cursor.java +++ b/core/src/main/java/google/registry/model/common/Cursor.java @@ -31,7 +31,6 @@ import google.registry.model.UnsafeSerializable; import google.registry.model.UpdateAutoTimestamp; import google.registry.model.annotations.InCrossTld; import google.registry.model.common.Cursor.CursorId; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.tld.Registry; import google.registry.persistence.VKey; import java.util.List; @@ -53,7 +52,7 @@ import org.joda.time.DateTime; @javax.persistence.Entity @IdClass(CursorId.class) @InCrossTld -public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity, UnsafeSerializable { +public class Cursor extends ImmutableObject implements UnsafeSerializable { /** The scope of a global cursor. A global cursor is a cursor that is not specific to one tld. */ public static final String GLOBAL = "GLOBAL"; diff --git a/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java b/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java index bafe6d7bf..a99a873a1 100644 --- a/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java +++ b/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java @@ -27,7 +27,6 @@ import google.registry.config.RegistryEnvironment; import google.registry.model.CacheUtils; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.common.TimedTransitionProperty.TimedTransition; -import google.registry.model.replay.SqlOnlyEntity; import java.time.Duration; import java.util.Arrays; import javax.persistence.Entity; @@ -42,7 +41,7 @@ import org.joda.time.DateTime; */ @DeleteAfterMigration @Entity -public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements SqlOnlyEntity { +public class DatabaseMigrationStateSchedule extends CrossTldSingleton { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -218,7 +217,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements MigrationState.DATASTORE_ONLY, "migrationTransitionMap must start with DATASTORE_ONLY"); validateTransitionAtCurrentTime(transitions); - jpaTm().putIgnoringReadOnlyWithoutBackup(new DatabaseMigrationStateSchedule(transitions)); + jpaTm().putWithoutBackup(new DatabaseMigrationStateSchedule(transitions)); CACHE.invalidateAll(); } diff --git a/core/src/main/java/google/registry/model/common/EntityGroupRoot.java b/core/src/main/java/google/registry/model/common/EntityGroupRoot.java index 6009ad454..bbd4f273c 100644 --- a/core/src/main/java/google/registry/model/common/EntityGroupRoot.java +++ b/core/src/main/java/google/registry/model/common/EntityGroupRoot.java @@ -20,7 +20,6 @@ import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Id; import google.registry.model.BackupGroupRoot; import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.replay.DatastoreOnlyEntity; import javax.annotation.Nullable; /** @@ -39,7 +38,7 @@ import javax.annotation.Nullable; */ @Entity @DeleteAfterMigration -public class EntityGroupRoot extends BackupGroupRoot implements DatastoreOnlyEntity { +public class EntityGroupRoot extends BackupGroupRoot { @SuppressWarnings("unused") @Id diff --git a/core/src/main/java/google/registry/model/contact/ContactHistory.java b/core/src/main/java/google/registry/model/contact/ContactHistory.java index d59d5fe03..1683133fa 100644 --- a/core/src/main/java/google/registry/model/contact/ContactHistory.java +++ b/core/src/main/java/google/registry/model/contact/ContactHistory.java @@ -20,8 +20,6 @@ import google.registry.model.EppResource; import google.registry.model.ImmutableObject; import google.registry.model.UnsafeSerializable; import google.registry.model.contact.ContactHistory.ContactHistoryId; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import java.io.Serializable; @@ -58,7 +56,7 @@ import javax.persistence.PostLoad; @EntitySubclass @Access(AccessType.FIELD) @IdClass(ContactHistoryId.class) -public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable { +public class ContactHistory extends HistoryEntry implements UnsafeSerializable { // Store ContactBase instead of ContactResource so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects @@ -128,12 +126,6 @@ public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSer } } - // In Datastore, save as a HistoryEntry object regardless of this object's type - @Override - public Optional toDatastoreEntity() { - return Optional.of(asHistoryEntry()); - } - /** Class to represent the composite primary key of {@link ContactHistory} entity. */ public static class ContactHistoryId extends ImmutableObject implements Serializable { diff --git a/core/src/main/java/google/registry/model/contact/ContactResource.java b/core/src/main/java/google/registry/model/contact/ContactResource.java index 2c31c98a5..59c98f772 100644 --- a/core/src/main/java/google/registry/model/contact/ContactResource.java +++ b/core/src/main/java/google/registry/model/contact/ContactResource.java @@ -19,7 +19,6 @@ import com.googlecode.objectify.annotation.Entity; import google.registry.model.EppResource.ForeignKeyedEppResource; import google.registry.model.annotations.ExternalMessagingName; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import javax.persistence.Access; @@ -46,8 +45,7 @@ import org.joda.time.DateTime; @ExternalMessagingName("contact") @WithStringVKey @Access(AccessType.FIELD) -public class ContactResource extends ContactBase - implements DatastoreAndSqlEntity, ForeignKeyedEppResource { +public class ContactResource extends ContactBase implements ForeignKeyedEppResource { @Override public VKey createVKey() { diff --git a/core/src/main/java/google/registry/model/domain/DomainBase.java b/core/src/main/java/google/registry/model/domain/DomainBase.java index 0950daddb..c94ca4987 100644 --- a/core/src/main/java/google/registry/model/domain/DomainBase.java +++ b/core/src/main/java/google/registry/model/domain/DomainBase.java @@ -21,7 +21,6 @@ import google.registry.model.annotations.ExternalMessagingName; import google.registry.model.annotations.ReportedOn; import google.registry.model.domain.secdns.DelegationSignerData; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import java.util.Set; @@ -74,8 +73,7 @@ import org.joda.time.DateTime; @WithStringVKey @ExternalMessagingName("domain") @Access(AccessType.FIELD) -public class DomainBase extends DomainContent - implements DatastoreAndSqlEntity, ForeignKeyedEppResource { +public class DomainBase extends DomainContent implements ForeignKeyedEppResource { @Override @javax.persistence.Id diff --git a/core/src/main/java/google/registry/model/domain/DomainHistory.java b/core/src/main/java/google/registry/model/domain/DomainHistory.java index 3cb3e633a..34cb44fac 100644 --- a/core/src/main/java/google/registry/model/domain/DomainHistory.java +++ b/core/src/main/java/google/registry/model/domain/DomainHistory.java @@ -27,8 +27,6 @@ import google.registry.model.domain.GracePeriod.GracePeriodHistory; import google.registry.model.domain.secdns.DelegationSignerData; import google.registry.model.domain.secdns.DomainDsDataHistory; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.DomainTransactionRecord; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; @@ -80,7 +78,7 @@ import org.hibernate.Hibernate; @EntitySubclass @Access(AccessType.FIELD) @IdClass(DomainHistoryId.class) -public class DomainHistory extends HistoryEntry implements SqlEntity { +public class DomainHistory extends HistoryEntry { // Store DomainContent instead of DomainBase so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects @@ -295,12 +293,6 @@ public class DomainHistory extends HistoryEntry implements SqlEntity { } } - // In Datastore, save as a HistoryEntry object regardless of this object's type - @Override - public Optional toDatastoreEntity() { - return Optional.of(asHistoryEntry()); - } - private static void fillAuxiliaryFieldsFromDomain(DomainHistory domainHistory) { if (domainHistory.domainContent != null) { domainHistory.nsHosts = nullToEmptyImmutableCopy(domainHistory.domainContent.nsHosts); diff --git a/core/src/main/java/google/registry/model/domain/GracePeriod.java b/core/src/main/java/google/registry/model/domain/GracePeriod.java index 486f42880..9db7171f6 100644 --- a/core/src/main/java/google/registry/model/domain/GracePeriod.java +++ b/core/src/main/java/google/registry/model/domain/GracePeriod.java @@ -24,8 +24,6 @@ import google.registry.model.billing.BillingEvent; import google.registry.model.billing.BillingEvent.Recurring; import google.registry.model.domain.DomainHistory.DomainHistoryId; import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.BillingVKey.BillingEventVKey; import google.registry.persistence.BillingVKey.BillingRecurrenceVKey; import google.registry.persistence.VKey; @@ -52,7 +50,7 @@ import org.joda.time.DateTime; @Index(columnList = "billing_event_id"), @Index(columnList = "billing_recurrence_id") }) -public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntity { +public class GracePeriod extends GracePeriodBase { @Id @Access(AccessType.PROPERTY) @@ -197,7 +195,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit /** Entity class to represent a historic {@link GracePeriod}. */ @Entity(name = "GracePeriodHistory") @Table(indexes = @Index(columnList = "domainRepoId")) - public static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity { + public static class GracePeriodHistory extends GracePeriodBase { @Id Long gracePeriodHistoryRevisionId; /** ID for the associated {@link DomainHistory} entity. */ diff --git a/core/src/main/java/google/registry/model/domain/RegistryLock.java b/core/src/main/java/google/registry/model/domain/RegistryLock.java index cf1adb511..daedb8e9f 100644 --- a/core/src/main/java/google/registry/model/domain/RegistryLock.java +++ b/core/src/main/java/google/registry/model/domain/RegistryLock.java @@ -23,7 +23,6 @@ import google.registry.model.Buildable; import google.registry.model.CreateAutoTimestamp; import google.registry.model.ImmutableObject; import google.registry.model.UpdateAutoTimestamp; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.util.DateTimeUtils; import java.time.ZonedDateTime; import java.util.Optional; @@ -76,7 +75,7 @@ import org.joda.time.Duration; @Index(name = "idx_registry_lock_verification_code", columnList = "verificationCode"), @Index(name = "idx_registry_lock_registrar_id", columnList = "registrarId") }) -public final class RegistryLock extends ImmutableObject implements Buildable, SqlOnlyEntity { +public final class RegistryLock extends ImmutableObject implements Buildable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java b/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java index 4d1e646a3..bcdf392dc 100644 --- a/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java +++ b/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java @@ -19,7 +19,6 @@ import static google.registry.model.IdService.allocateId; import google.registry.model.UnsafeSerializable; import google.registry.model.domain.DomainHistory; import google.registry.model.domain.DomainHistory.DomainHistoryId; -import google.registry.model.replay.SqlOnlyEntity; import javax.persistence.Access; import javax.persistence.AccessType; import javax.persistence.Column; @@ -28,8 +27,7 @@ import javax.persistence.Id; /** Entity class to represent a historic {@link DelegationSignerData}. */ @Entity -public class DomainDsDataHistory extends DomainDsDataBase - implements SqlOnlyEntity, UnsafeSerializable { +public class DomainDsDataHistory extends DomainDsDataBase implements UnsafeSerializable { @Id Long dsDataHistoryRevisionId; diff --git a/core/src/main/java/google/registry/model/domain/token/AllocationToken.java b/core/src/main/java/google/registry/model/domain/token/AllocationToken.java index aab73084b..5c96f25b1 100644 --- a/core/src/main/java/google/registry/model/domain/token/AllocationToken.java +++ b/core/src/main/java/google/registry/model/domain/token/AllocationToken.java @@ -47,7 +47,6 @@ import google.registry.model.billing.BillingEvent.RenewalPriceBehavior; import google.registry.model.common.TimedTransitionProperty; import google.registry.model.common.TimedTransitionProperty.TimeMapper; import google.registry.model.common.TimedTransitionProperty.TimedTransition; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.DomainHistoryVKey; import google.registry.persistence.VKey; @@ -80,7 +79,7 @@ import org.joda.time.DateTime; @javax.persistence.Index(columnList = "tokenType"), @javax.persistence.Index(columnList = "redemption_domain_repo_id") }) -public class AllocationToken extends BackupGroupRoot implements Buildable, DatastoreAndSqlEntity { +public class AllocationToken extends BackupGroupRoot implements Buildable { // Promotions should only move forward, and ENDED / CANCELLED are terminal states. private static final ImmutableMultimap VALID_TOKEN_STATUS_TRANSITIONS = diff --git a/core/src/main/java/google/registry/model/host/HostHistory.java b/core/src/main/java/google/registry/model/host/HostHistory.java index ec9b541bf..d7c3a879e 100644 --- a/core/src/main/java/google/registry/model/host/HostHistory.java +++ b/core/src/main/java/google/registry/model/host/HostHistory.java @@ -20,8 +20,6 @@ import google.registry.model.EppResource; import google.registry.model.ImmutableObject; import google.registry.model.UnsafeSerializable; import google.registry.model.host.HostHistory.HostHistoryId; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import java.io.Serializable; @@ -59,7 +57,7 @@ import javax.persistence.PostLoad; @EntitySubclass @Access(AccessType.FIELD) @IdClass(HostHistoryId.class) -public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable { +public class HostHistory extends HistoryEntry implements UnsafeSerializable { // Store HostBase instead of HostResource so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects @@ -128,12 +126,6 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial } } - // In Datastore, save as a HistoryEntry object regardless of this object's type - @Override - public Optional toDatastoreEntity() { - return Optional.of(asHistoryEntry()); - } - /** Class to represent the composite primary key of {@link HostHistory} entity. */ public static class HostHistoryId extends ImmutableObject implements Serializable { diff --git a/core/src/main/java/google/registry/model/host/HostResource.java b/core/src/main/java/google/registry/model/host/HostResource.java index aa952b5af..7436d8086 100644 --- a/core/src/main/java/google/registry/model/host/HostResource.java +++ b/core/src/main/java/google/registry/model/host/HostResource.java @@ -19,7 +19,6 @@ import com.googlecode.objectify.annotation.Entity; import google.registry.model.EppResource.ForeignKeyedEppResource; import google.registry.model.annotations.ExternalMessagingName; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import javax.persistence.Access; @@ -55,8 +54,7 @@ import javax.persistence.AccessType; @ExternalMessagingName("host") @WithStringVKey @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property) -public class HostResource extends HostBase - implements DatastoreAndSqlEntity, ForeignKeyedEppResource { +public class HostResource extends HostBase implements ForeignKeyedEppResource { @Override @javax.persistence.Id diff --git a/core/src/main/java/google/registry/model/index/EppResourceIndex.java b/core/src/main/java/google/registry/model/index/EppResourceIndex.java index 5a577d9ba..8b78a3bc5 100644 --- a/core/src/main/java/google/registry/model/index/EppResourceIndex.java +++ b/core/src/main/java/google/registry/model/index/EppResourceIndex.java @@ -26,14 +26,13 @@ import google.registry.model.BackupGroupRoot; import google.registry.model.EppResource; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.persistence.VKey; /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */ @ReportedOn @Entity @DeleteAfterMigration -public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEntity { +public class EppResourceIndex extends BackupGroupRoot { @Id String id; diff --git a/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java b/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java index ad41ff035..2f33d4299 100644 --- a/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java +++ b/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java @@ -25,13 +25,12 @@ import google.registry.model.EppResource; import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.VirtualEntity; -import google.registry.model.replay.DatastoreOnlyEntity; /** A virtual entity to represent buckets to which EppResourceIndex objects are randomly added. */ @Entity @VirtualEntity @DeleteAfterMigration -public class EppResourceIndexBucket extends ImmutableObject implements DatastoreOnlyEntity { +public class EppResourceIndexBucket extends ImmutableObject { @SuppressWarnings("unused") @Id diff --git a/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java b/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java index 0b7e9ee3a..2ac3b6907 100644 --- a/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java +++ b/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java @@ -50,7 +50,6 @@ import google.registry.model.annotations.ReportedOn; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DomainBase; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.persistence.VKey; import google.registry.persistence.transaction.CriteriaQueryBuilder; import google.registry.persistence.transaction.JpaTransactionManager; @@ -74,20 +73,17 @@ public abstract class ForeignKeyIndex extends BackupGroup /** The {@link ForeignKeyIndex} type for {@link ContactResource} entities. */ @ReportedOn @Entity - public static class ForeignKeyContactIndex extends ForeignKeyIndex - implements DatastoreOnlyEntity {} + public static class ForeignKeyContactIndex extends ForeignKeyIndex {} /** The {@link ForeignKeyIndex} type for {@link DomainBase} entities. */ @ReportedOn @Entity - public static class ForeignKeyDomainIndex extends ForeignKeyIndex - implements DatastoreOnlyEntity {} + public static class ForeignKeyDomainIndex extends ForeignKeyIndex {} /** The {@link ForeignKeyIndex} type for {@link HostResource} entities. */ @ReportedOn @Entity - public static class ForeignKeyHostIndex extends ForeignKeyIndex - implements DatastoreOnlyEntity {} + public static class ForeignKeyHostIndex extends ForeignKeyIndex {} private static final ImmutableBiMap< Class, Class>> diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java b/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java index c8e53747f..9221304c9 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java @@ -34,7 +34,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.util.NonFinalForTesting; import java.util.Random; import java.util.function.Supplier; @@ -53,7 +52,7 @@ import org.joda.time.DateTime; @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogBucket extends ImmutableObject implements Buildable, DatastoreOnlyEntity { +public class CommitLogBucket extends ImmutableObject implements Buildable { /** * Ranges from 1 to {@link RegistryConfig#getCommitLogBucketCount()}, inclusive; starts at 1 since diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java index 867f5f804..55cc57184 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java @@ -28,7 +28,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -47,7 +46,7 @@ import org.joda.time.DateTime; @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogCheckpoint extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogCheckpoint extends ImmutableObject { /** Shared singleton parent entity for commit log checkpoints. */ @Parent diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java index e173f61d6..b25b9ebea 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java @@ -24,14 +24,13 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import org.joda.time.DateTime; /** Singleton parent entity for all commit log checkpoints. */ @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogCheckpointRoot extends ImmutableObject { public static final long SINGLETON_ID = 1; // There is always exactly one of these. diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java b/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java index 8ad8248f7..78b1528e6 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java @@ -26,7 +26,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import java.util.LinkedHashSet; import java.util.Set; import org.joda.time.DateTime; @@ -41,7 +40,7 @@ import org.joda.time.DateTime; @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogManifest extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogManifest extends ImmutableObject { /** Commit log manifests are parented on a random bucket. */ @Parent diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java b/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java index 87df17d7e..66f02aa38 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java @@ -30,13 +30,12 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; /** Representation of a saved entity in a {@link CommitLogManifest} (not deletes). */ @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogMutation extends ImmutableObject { /** The manifest this belongs to. */ @Parent diff --git a/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java b/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java index e8187d2f3..94ab71043 100644 --- a/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java +++ b/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java @@ -38,8 +38,6 @@ import google.registry.model.annotations.InCrossTld; import google.registry.model.contact.ContactHistory; import google.registry.model.domain.DomainHistory; import google.registry.model.host.HostHistory; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import google.registry.persistence.transaction.QueryComposer; @@ -357,28 +355,6 @@ public class DatastoreTransactionManager implements TransactionManager { return true; } - @Override - public void putIgnoringReadOnlyWithoutBackup(Object entity) { - syncIfTransactionless( - getOfy().saveIgnoringReadOnlyWithoutBackup().entities(toDatastoreEntity(entity))); - } - - @Override - public void deleteIgnoringReadOnlyWithoutBackup(VKey key) { - syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithoutBackup().key(key.getOfyKey())); - } - - /** Performs the write ignoring read-only restrictions and also writes commit logs. */ - public void putIgnoringReadOnlyWithBackup(Object entity) { - syncIfTransactionless( - getOfy().saveIgnoringReadOnlyWithBackup().entities(toDatastoreEntity(entity))); - } - - /** Performs the delete ignoring read-only restrictions and also writes commit logs. */ - public void deleteIgnoringReadOnlyWithBackup(VKey key) { - syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithBackup().key(key.getOfyKey())); - } - /** * Executes the given {@link Result} instance synchronously if not in a transaction. * @@ -413,22 +389,14 @@ public class DatastoreTransactionManager implements TransactionManager { return toSqlEntity(getOfy().load().key(key.getOfyKey()).now()); } - /** - * Converts a possible {@link SqlEntity} to a {@link DatastoreEntity}. - * - *

One example is that this would convert a {@link DomainHistory} to a {@link HistoryEntry}. - */ + /** Converts a possible {@link HistoryEntry} child to a {@link HistoryEntry}. */ private static Object toDatastoreEntity(@Nullable Object obj) { - if (obj instanceof SqlEntity) { - Optional possibleDatastoreEntity = ((SqlEntity) obj).toDatastoreEntity(); - if (possibleDatastoreEntity.isPresent()) { - return possibleDatastoreEntity.get(); - } + if (obj instanceof HistoryEntry) { + return ((HistoryEntry) obj).asHistoryEntry(); } return obj; } - /** Converts many possible {@link SqlEntity} objects to {@link DatastoreEntity} objects. */ private static ImmutableList toDatastoreEntities(ImmutableCollection collection) { return collection.stream() .map(DatastoreTransactionManager::toDatastoreEntity) @@ -436,21 +404,15 @@ public class DatastoreTransactionManager implements TransactionManager { } /** - * Converts an object to the corresponding {@link SqlEntity} if necessary and possible. + * Converts an object to the corresponding child {@link HistoryEntry} if necessary and possible. * *

This should be used when returning objects from Datastore to make sure they reflect the most * recent type of the object in question. */ @SuppressWarnings("unchecked") public static T toSqlEntity(@Nullable T obj) { - // NB: The Key of the object in question may not necessarily be the resulting class that we - // wish to have. For example, because all *History classes are @EntitySubclasses, their Keys - // will have type HistoryEntry -- even if you create them based off the *History class. - if (obj instanceof DatastoreEntity && !(obj instanceof SqlEntity)) { - Optional possibleSqlEntity = ((DatastoreEntity) obj).toSqlEntity(); - if (possibleSqlEntity.isPresent()) { - return (T) possibleSqlEntity.get(); - } + if (obj instanceof HistoryEntry) { + return (T) ((HistoryEntry) obj).toChildHistoryEntity(); } return obj; } diff --git a/core/src/main/java/google/registry/model/ofy/Ofy.java b/core/src/main/java/google/registry/model/ofy/Ofy.java index eafa8c1d6..9f56f864d 100644 --- a/core/src/main/java/google/registry/model/ofy/Ofy.java +++ b/core/src/main/java/google/registry/model/ofy/Ofy.java @@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Maps.uniqueIndex; import static com.googlecode.objectify.ObjectifyService.ofy; import static google.registry.config.RegistryConfig.getBaseOfyRetryDuration; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; import static google.registry.util.CollectionUtils.union; import com.google.appengine.api.datastore.DatastoreFailureException; @@ -134,7 +133,6 @@ public class Ofy { *

We only allow this in transactions so commit logs can be written in tandem with the delete. */ public Deleter delete() { - assertNotReadOnlyMode(); return deleteIgnoringReadOnlyWithBackup(); } @@ -144,7 +142,6 @@ public class Ofy { *

No backups get written. */ public Deleter deleteWithoutBackup() { - assertNotReadOnlyMode(); return deleteIgnoringReadOnlyWithoutBackup(); } @@ -155,7 +152,6 @@ public class Ofy { *

We only allow this in transactions so commit logs can be written in tandem with the save. */ public Saver save() { - assertNotReadOnlyMode(); return saveIgnoringReadOnlyWithBackup(); } @@ -165,7 +161,6 @@ public class Ofy { *

No backups get written. */ public Saver saveWithoutBackup() { - assertNotReadOnlyMode(); return saveIgnoringReadOnlyWithoutBackup(); } diff --git a/core/src/main/java/google/registry/model/poll/PollMessage.java b/core/src/main/java/google/registry/model/poll/PollMessage.java index 7491c4551..3fce7a691 100644 --- a/core/src/main/java/google/registry/model/poll/PollMessage.java +++ b/core/src/main/java/google/registry/model/poll/PollMessage.java @@ -44,7 +44,6 @@ import google.registry.model.host.HostResource; import google.registry.model.poll.PendingActionNotificationResponse.ContactPendingActionNotificationResponse; import google.registry.model.poll.PendingActionNotificationResponse.DomainPendingActionNotificationResponse; import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.model.transfer.TransferData.TransferServerApproveEntity; import google.registry.model.transfer.TransferResponse; @@ -99,7 +98,7 @@ import org.joda.time.DateTime; @javax.persistence.Index(columnList = "eventTime") }) public abstract class PollMessage extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity, UnsafeSerializable { + implements Buildable, TransferServerApproveEntity, UnsafeSerializable { /** Entity id. */ @Id diff --git a/core/src/main/java/google/registry/model/rde/RdeRevision.java b/core/src/main/java/google/registry/model/rde/RdeRevision.java index 659c8ee48..0a2d06bd8 100644 --- a/core/src/main/java/google/registry/model/rde/RdeRevision.java +++ b/core/src/main/java/google/registry/model/rde/RdeRevision.java @@ -27,7 +27,6 @@ import com.googlecode.objectify.annotation.Ignore; import google.registry.model.BackupGroupRoot; import google.registry.model.ImmutableObject; import google.registry.model.rde.RdeRevision.RdeRevisionId; -import google.registry.model.replay.NonReplicatedEntity; import google.registry.persistence.VKey; import google.registry.persistence.converter.LocalDateConverter; import java.io.Serializable; @@ -51,7 +50,7 @@ import org.joda.time.LocalDate; @Entity @javax.persistence.Entity @IdClass(RdeRevisionId.class) -public final class RdeRevision extends BackupGroupRoot implements NonReplicatedEntity { +public final class RdeRevision extends BackupGroupRoot { /** String triplet of tld, date, and mode, e.g. {@code soy_2015-09-01_full}. */ @Id @Transient String id; diff --git a/core/src/main/java/google/registry/model/registrar/Registrar.java b/core/src/main/java/google/registry/model/registrar/Registrar.java index efb71ff1d..70c5e3997 100644 --- a/core/src/main/java/google/registry/model/registrar/Registrar.java +++ b/core/src/main/java/google/registry/model/registrar/Registrar.java @@ -79,7 +79,6 @@ import google.registry.model.annotations.InCrossTld; import google.registry.model.annotations.ReportedOn; import google.registry.model.common.EntityGroupRoot; import google.registry.model.registrar.Registrar.BillingAccountEntry.CurrencyMapper; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.tld.Registry; import google.registry.model.tld.Registry.TldType; import google.registry.persistence.VKey; @@ -119,7 +118,7 @@ import org.joda.time.DateTime; }) @InCrossTld public class Registrar extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable { + implements Buildable, Jsonifiable, UnsafeSerializable { /** Represents the type of a registrar entity. */ public enum Type { diff --git a/core/src/main/java/google/registry/model/registrar/RegistrarContact.java b/core/src/main/java/google/registry/model/registrar/RegistrarContact.java index d4aefc041..bbab8b6ba 100644 --- a/core/src/main/java/google/registry/model/registrar/RegistrarContact.java +++ b/core/src/main/java/google/registry/model/registrar/RegistrarContact.java @@ -50,7 +50,6 @@ import google.registry.model.UnsafeSerializable; import google.registry.model.annotations.InCrossTld; import google.registry.model.annotations.ReportedOn; import google.registry.model.registrar.RegistrarContact.RegistrarPocId; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import java.io.Serializable; import java.util.Arrays; @@ -82,8 +81,7 @@ import javax.persistence.Transient; }) @IdClass(RegistrarPocId.class) @InCrossTld -public class RegistrarContact extends ImmutableObject - implements DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable { +public class RegistrarContact extends ImmutableObject implements Jsonifiable, UnsafeSerializable { @Parent @Transient Key parent; diff --git a/core/src/main/java/google/registry/model/replay/DatastoreAndSqlEntity.java b/core/src/main/java/google/registry/model/replay/DatastoreAndSqlEntity.java deleted file mode 100644 index 8fb61dc90..000000000 --- a/core/src/main/java/google/registry/model/replay/DatastoreAndSqlEntity.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** An entity that has the same Java object representation in SQL and Datastore. */ -@DeleteAfterMigration -public interface DatastoreAndSqlEntity extends DatastoreEntity, SqlEntity { - - @Override - default Optional toDatastoreEntity() { - return Optional.of(this); - } - - @Override - default Optional toSqlEntity() { - return Optional.of(this); - } -} diff --git a/core/src/main/java/google/registry/model/replay/DatastoreEntity.java b/core/src/main/java/google/registry/model/replay/DatastoreEntity.java deleted file mode 100644 index 7824455b8..000000000 --- a/core/src/main/java/google/registry/model/replay/DatastoreEntity.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** - * An object that can be stored in Datastore and serialized using Objectify's {@link - * com.googlecode.objectify.cmd.Saver#toEntity(Object)} code. - * - *

This is used when replaying {@link google.registry.model.ofy.CommitLogManifest}s to import - * transactions and data into the secondary SQL store during the first, Datastore-primary, phase of - * the migration. - */ -@DeleteAfterMigration -public interface DatastoreEntity { - - Optional toSqlEntity(); -} diff --git a/core/src/main/java/google/registry/model/replay/DatastoreOnlyEntity.java b/core/src/main/java/google/registry/model/replay/DatastoreOnlyEntity.java deleted file mode 100644 index 842e31a51..000000000 --- a/core/src/main/java/google/registry/model/replay/DatastoreOnlyEntity.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** An entity that is only stored in Datastore, that should not be replayed to SQL. */ -@DeleteAfterMigration -public interface DatastoreOnlyEntity extends DatastoreEntity { - @Override - default Optional toSqlEntity() { - return Optional.empty(); - } -} diff --git a/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java b/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java index 678e24c8b..51a1a503a 100644 --- a/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java +++ b/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java @@ -27,7 +27,7 @@ import google.registry.model.annotations.DeleteAfterMigration; /** Datastore entity to keep track of the last SQL transaction imported into the datastore. */ @Entity @DeleteAfterMigration -public class LastSqlTransaction extends ImmutableObject implements DatastoreOnlyEntity { +public class LastSqlTransaction extends ImmutableObject { /** The key for this singleton. */ public static final Key KEY = Key.create(LastSqlTransaction.class, 1); diff --git a/core/src/main/java/google/registry/model/replay/NonReplicatedEntity.java b/core/src/main/java/google/registry/model/replay/NonReplicatedEntity.java deleted file mode 100644 index 7878a3132..000000000 --- a/core/src/main/java/google/registry/model/replay/NonReplicatedEntity.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** - * Represents an entity that should not participate in asynchronous replication. - * - *

We expect that this is a result of the entity being dually-written. - */ -@DeleteAfterMigration -public interface NonReplicatedEntity extends DatastoreEntity, SqlEntity { - - @Override - default Optional toDatastoreEntity() { - return Optional.empty(); - } - - @Override - default Optional toSqlEntity() { - return Optional.empty(); - } -} diff --git a/core/src/main/java/google/registry/model/replay/ReplayGap.java b/core/src/main/java/google/registry/model/replay/ReplayGap.java index 921c9ea35..a850b54e4 100644 --- a/core/src/main/java/google/registry/model/replay/ReplayGap.java +++ b/core/src/main/java/google/registry/model/replay/ReplayGap.java @@ -35,7 +35,7 @@ import org.joda.time.DateTime; @DeleteAfterMigration @NotBackedUp(reason = TRANSIENT) @Entity -public class ReplayGap extends ImmutableObject implements DatastoreOnlyEntity { +public class ReplayGap extends ImmutableObject { @Id long transactionId; // We can't use a CreateAutoTimestamp here because this ends up getting persisted in an ofy diff --git a/core/src/main/java/google/registry/model/replay/SqlEntity.java b/core/src/main/java/google/registry/model/replay/SqlEntity.java deleted file mode 100644 index ea09469f7..000000000 --- a/core/src/main/java/google/registry/model/replay/SqlEntity.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** - * An object that can be stored in Cloud SQL using {@link - * javax.persistence.EntityManager#persist(Object)} - * - *

This will be used when replaying SQL transactions into Datastore, during the second, - * SQL-primary, phase of the migration from Datastore to SQL. - */ -@DeleteAfterMigration -public interface SqlEntity { - - Optional toDatastoreEntity(); - - /** Returns this entity's primary key field(s) in a string. */ - default String getPrimaryKeyString() { - return jpaTm() - .transact( - () -> - String.format( - "%s_%s", - this.getClass().getSimpleName(), - jpaTm() - .getEntityManager() - .getEntityManagerFactory() - .getPersistenceUnitUtil() - .getIdentifier(this))); - } -} diff --git a/core/src/main/java/google/registry/model/replay/SqlOnlyEntity.java b/core/src/main/java/google/registry/model/replay/SqlOnlyEntity.java deleted file mode 100644 index ac156b758..000000000 --- a/core/src/main/java/google/registry/model/replay/SqlOnlyEntity.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** An entity that is only stored in SQL, that should not be replayed to Datastore. */ -@DeleteAfterMigration -public interface SqlOnlyEntity extends SqlEntity { - @Override - default Optional toDatastoreEntity() { - return Optional.empty(); - } -} diff --git a/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java b/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java index 89f8cedf1..5ec529e3d 100644 --- a/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java +++ b/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java @@ -25,7 +25,7 @@ import org.joda.time.DateTime; @Entity @DeleteAfterMigration -public class SqlReplayCheckpoint extends CrossTldSingleton implements SqlOnlyEntity { +public class SqlReplayCheckpoint extends CrossTldSingleton { @Column(nullable = false) private DateTime lastReplayTime; @@ -43,6 +43,6 @@ public class SqlReplayCheckpoint extends CrossTldSingleton implements SqlOnlyEnt SqlReplayCheckpoint checkpoint = new SqlReplayCheckpoint(); checkpoint.lastReplayTime = lastReplayTime; // this will overwrite the existing object due to the constant revisionId - jpaTm().putIgnoringReadOnlyWithoutBackup(checkpoint); + jpaTm().put(checkpoint); } } diff --git a/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java b/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java index 1f9ccaf8d..52969cec5 100644 --- a/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java +++ b/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java @@ -24,7 +24,6 @@ import google.registry.model.Buildable; import google.registry.model.ImmutableObject; import google.registry.model.UnsafeSerializable; import google.registry.model.domain.DomainHistory.DomainHistoryId; -import google.registry.model.replay.DatastoreAndSqlEntity; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; @@ -47,7 +46,7 @@ import org.joda.time.DateTime; @Embed @Entity public class DomainTransactionRecord extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable { + implements Buildable, UnsafeSerializable { @Id @Ignore diff --git a/core/src/main/java/google/registry/model/reporting/HistoryEntry.java b/core/src/main/java/google/registry/model/reporting/HistoryEntry.java index 9cd6cef53..54f87f9ea 100644 --- a/core/src/main/java/google/registry/model/reporting/HistoryEntry.java +++ b/core/src/main/java/google/registry/model/reporting/HistoryEntry.java @@ -48,8 +48,6 @@ import google.registry.model.host.HostBase; import google.registry.model.host.HostHistory; import google.registry.model.host.HostHistory.HostHistoryId; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.persistence.VKey; import java.util.Optional; import java.util.Set; @@ -86,8 +84,7 @@ import org.joda.time.DateTime; @Entity @MappedSuperclass @Access(AccessType.FIELD) -public class HistoryEntry extends ImmutableObject - implements Buildable, DatastoreEntity, UnsafeSerializable { +public class HistoryEntry extends ImmutableObject implements Buildable, UnsafeSerializable { /** Represents the type of history entry. */ public enum Type { @@ -405,12 +402,6 @@ public class HistoryEntry extends ImmutableObject return resultEntity; } - // In SQL, save the child type - @Override - public Optional toSqlEntity() { - return Optional.of((SqlEntity) toChildHistoryEntity()); - } - /** Creates a {@link VKey} instance from a {@link Key} instance. */ public static VKey createVKey(Key key) { String repoId = key.getParent().getName(); diff --git a/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java b/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java index aec00a2db..5232df8cc 100644 --- a/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java +++ b/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java @@ -21,7 +21,6 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import com.google.common.collect.ImmutableSet; import google.registry.model.Buildable; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.util.DomainNameUtils; import java.util.Set; import javax.persistence.Column; @@ -40,7 +39,7 @@ import org.joda.time.LocalDate; @Index(name = "spec11threatmatch_tld_idx", columnList = "tld"), @Index(name = "spec11threatmatch_check_date_idx", columnList = "checkDate") }) -public class Spec11ThreatMatch extends ImmutableObject implements Buildable, SqlOnlyEntity { +public class Spec11ThreatMatch extends ImmutableObject implements Buildable { /** The type of threat detected. */ public enum ThreatType { diff --git a/core/src/main/java/google/registry/model/server/Lock.java b/core/src/main/java/google/registry/model/server/Lock.java index d908d3848..fedd47a0d 100644 --- a/core/src/main/java/google/registry/model/server/Lock.java +++ b/core/src/main/java/google/registry/model/server/Lock.java @@ -30,7 +30,6 @@ import com.googlecode.objectify.annotation.Id; import google.registry.model.ImmutableObject; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.persistence.transaction.TransactionManager; @@ -62,7 +61,7 @@ import org.joda.time.Duration; @javax.persistence.Entity @Table @IdClass(Lock.LockId.class) -public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Serializable { +public class Lock extends ImmutableObject implements Serializable { private static final long serialVersionUID = 756397280691684645L; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -293,7 +292,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri create(resourceName, scope, requestStatusChecker.getLogId(), now, leaseLength); // Locks are not parented under an EntityGroupRoot (so as to avoid write // contention) and don't need to be backed up. - transactionManager.putIgnoringReadOnlyWithoutBackup(newLock); + transactionManager.put(newLock); return AcquireResult.create(now, lock, newLock, lockState); }; @@ -325,7 +324,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri // Use deleteIgnoringReadOnly() so that we don't create a commit log entry for deleting // the lock. logger.atInfo().log("Deleting lock: %s", lockId); - transactionManager.deleteIgnoringReadOnlyWithoutBackup(key); + transactionManager.delete(key); lockMetrics.recordRelease( resourceName, diff --git a/core/src/main/java/google/registry/model/server/ServerSecret.java b/core/src/main/java/google/registry/model/server/ServerSecret.java index 91079967f..ef965e0f5 100644 --- a/core/src/main/java/google/registry/model/server/ServerSecret.java +++ b/core/src/main/java/google/registry/model/server/ServerSecret.java @@ -27,7 +27,6 @@ import google.registry.model.CacheUtils; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; import google.registry.model.common.CrossTldSingleton; -import google.registry.model.replay.NonReplicatedEntity; import java.nio.ByteBuffer; import java.util.Optional; import java.util.UUID; @@ -41,7 +40,7 @@ import javax.persistence.Transient; @Unindex @NotBackedUp(reason = Reason.AUTO_GENERATED) // TODO(b/27427316): Replace this with an entry in KMSKeyring -public class ServerSecret extends CrossTldSingleton implements NonReplicatedEntity { +public class ServerSecret extends CrossTldSingleton { /** * Cache of the singleton ServerSecret instance that creates it if not present. diff --git a/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java b/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java index c498119fb..cf5867d65 100644 --- a/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java +++ b/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java @@ -21,7 +21,6 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import java.util.Map; import javax.persistence.CollectionTable; import javax.persistence.Column; @@ -46,7 +45,7 @@ import org.joda.time.DateTime; * functional specifications - SMD Revocation List */ @Entity -public class SignedMarkRevocationList extends ImmutableObject implements SqlOnlyEntity { +public class SignedMarkRevocationList extends ImmutableObject { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/model/tld/Registry.java b/core/src/main/java/google/registry/model/tld/Registry.java index 2d05e3cd3..18435dc80 100644 --- a/core/src/main/java/google/registry/model/tld/Registry.java +++ b/core/src/main/java/google/registry/model/tld/Registry.java @@ -61,7 +61,6 @@ import google.registry.model.common.TimedTransitionProperty; import google.registry.model.common.TimedTransitionProperty.TimedTransition; import google.registry.model.domain.fee.BaseFee.FeeType; import google.registry.model.domain.fee.Fee; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.tld.label.PremiumList; import google.registry.model.tld.label.ReservedList; import google.registry.persistence.VKey; @@ -90,8 +89,7 @@ import org.joda.time.Duration; @Entity @javax.persistence.Entity(name = "Tld") @InCrossTld -public class Registry extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable { +public class Registry extends ImmutableObject implements Buildable, UnsafeSerializable { @Parent @Transient Key parent = getCrossTldKey(); diff --git a/core/src/main/java/google/registry/model/tld/label/PremiumList.java b/core/src/main/java/google/registry/model/tld/label/PremiumList.java index 4c0487fad..5abb1539d 100644 --- a/core/src/main/java/google/registry/model/tld/label/PremiumList.java +++ b/core/src/main/java/google/registry/model/tld/label/PremiumList.java @@ -25,7 +25,6 @@ import com.google.common.hash.BloomFilter; import google.registry.model.Buildable; import google.registry.model.ImmutableObject; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.tld.Registry; import google.registry.model.tld.label.PremiumList.PremiumEntry; import java.io.Serializable; @@ -53,8 +52,7 @@ import org.joda.money.Money; @ReportedOn @javax.persistence.Entity @Table(indexes = {@Index(columnList = "name", name = "premiumlist_name_idx")}) -public final class PremiumList extends BaseDomainLabelList - implements SqlOnlyEntity { +public final class PremiumList extends BaseDomainLabelList { @Column(nullable = false) CurrencyUnit currency; @@ -120,7 +118,7 @@ public final class PremiumList extends BaseDomainLabelList - implements Buildable, SqlOnlyEntity, Serializable { + implements Buildable, Serializable { @ImmutableObject.Insignificant @javax.persistence.Id Long revisionId; diff --git a/core/src/main/java/google/registry/model/tld/label/ReservedList.java b/core/src/main/java/google/registry/model/tld/label/ReservedList.java index 7d7d0e9ed..75f1ee129 100644 --- a/core/src/main/java/google/registry/model/tld/label/ReservedList.java +++ b/core/src/main/java/google/registry/model/tld/label/ReservedList.java @@ -33,7 +33,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.UncheckedExecutionException; import google.registry.model.Buildable; import google.registry.model.CacheUtils; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.tld.Registry; import google.registry.model.tld.label.DomainLabelMetrics.MetricsReservedListMatch; import java.io.Serializable; @@ -61,8 +60,7 @@ import org.joda.time.DateTime; @javax.persistence.Entity @Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")}) public final class ReservedList - extends BaseDomainLabelList - implements SqlOnlyEntity { + extends BaseDomainLabelList { /** * Mapping from domain name to its reserved list info. @@ -88,7 +86,7 @@ public final class ReservedList /** * Hibernate hook called on the insert of a new ReservedList. Stores the associated {@link - * ReservedEntry}'s. + * ReservedListEntry}'s. * *

We need to persist the list entries, but only on the initial insert (not on update) since * the entries themselves never get changed, so we only annotate it with {@link PostPersist}, not @@ -113,7 +111,7 @@ public final class ReservedList */ @javax.persistence.Entity(name = "ReservedEntry") public static class ReservedListEntry extends DomainLabelEntry - implements Buildable, SqlOnlyEntity, Serializable { + implements Buildable, Serializable { @Insignificant @Id Long revisionId; diff --git a/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java b/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java index cc45933a0..fa407fb16 100644 --- a/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java +++ b/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java @@ -15,7 +15,6 @@ package google.registry.model.tmch; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import java.io.Serializable; import javax.persistence.Column; import javax.persistence.Entity; @@ -30,7 +29,7 @@ import javax.persistence.Id; * work. */ @Entity(name = "ClaimsEntry") -class ClaimsEntry extends ImmutableObject implements SqlOnlyEntity, Serializable { +class ClaimsEntry extends ImmutableObject implements Serializable { @Id private Long revisionId; @Id private String domainLabel; diff --git a/core/src/main/java/google/registry/model/tmch/ClaimsList.java b/core/src/main/java/google/registry/model/tmch/ClaimsList.java index ba9576dae..ec688a5b6 100644 --- a/core/src/main/java/google/registry/model/tmch/ClaimsList.java +++ b/core/src/main/java/google/registry/model/tmch/ClaimsList.java @@ -23,7 +23,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import com.google.common.collect.ImmutableMap; import google.registry.model.CreateAutoTimestamp; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.tld.label.ReservedList.ReservedListEntry; import java.util.Map; import java.util.Optional; @@ -52,7 +51,7 @@ import org.joda.time.DateTime; */ @Entity(name = "ClaimsList") @Table -public class ClaimsList extends ImmutableObject implements SqlOnlyEntity { +public class ClaimsList extends ImmutableObject { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/model/tmch/TmchCrl.java b/core/src/main/java/google/registry/model/tmch/TmchCrl.java index dae660667..5f2fb6b16 100644 --- a/core/src/main/java/google/registry/model/tmch/TmchCrl.java +++ b/core/src/main/java/google/registry/model/tmch/TmchCrl.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import google.registry.model.common.CrossTldSingleton; -import google.registry.model.replay.SqlOnlyEntity; import java.util.Optional; import javax.annotation.concurrent.Immutable; import javax.persistence.Column; @@ -27,7 +26,7 @@ import org.joda.time.DateTime; /** Singleton for ICANN's TMCH CA certificate revocation list (CRL). */ @javax.persistence.Entity @Immutable -public final class TmchCrl extends CrossTldSingleton implements SqlOnlyEntity { +public final class TmchCrl extends CrossTldSingleton { @Column(name = "certificateRevocations", nullable = false) String crl; diff --git a/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java b/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java index 1ab51d66d..9696976af 100644 --- a/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java +++ b/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java @@ -25,7 +25,6 @@ import com.google.common.collect.Ordering; import com.googlecode.objectify.Key; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.ofy.CommitLogManifest; -import google.registry.persistence.transaction.Transaction; import org.joda.time.DateTime; /** @@ -69,12 +68,6 @@ public final class CommitLogRevisionsTranslatorFactory @Override ImmutableSortedMap> transformBeforeSave( ImmutableSortedMap> revisions) { - - // Don't do anything if we're just doing object serialization. - if (Transaction.inSerializationMode()) { - return revisions; - } - DateTime now = ofyTm().getTransactionTime(); DateTime threshold = now.minus(getCommitLogDatastoreRetention()); DateTime preThresholdTime = firstNonNull(revisions.floorKey(threshold), START_OF_TIME); diff --git a/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java b/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java index 94d29c634..6bd1636af 100644 --- a/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java +++ b/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java @@ -14,13 +14,10 @@ package google.registry.model.translators; -import static com.google.common.base.Preconditions.checkState; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; import static org.joda.time.DateTimeZone.UTC; import google.registry.model.UpdateAutoTimestamp; -import google.registry.persistence.transaction.Transaction; import java.util.Date; import org.joda.time.DateTime; @@ -49,19 +46,6 @@ public class UpdateAutoTimestampTranslatorFactory /** Save a timestamp, setting it to the current time. */ @Override public Date saveValue(UpdateAutoTimestamp pojoValue) { - - // If we're in the course of Transaction serialization, we have to use the transaction time - // here and the JPA transaction manager which is what will ultimately be saved during the - // commit. - // Note that this branch doesn't respect "auto update disabled", as this state is - // specifically to address replay, so we add a runtime check for this. - if (Transaction.inSerializationMode()) { - checkState( - UpdateAutoTimestamp.autoUpdateEnabled(), - "Auto-update disabled during transaction serialization."); - return jpaTm().getTransactionTime().toDate(); - } - return UpdateAutoTimestamp.autoUpdateEnabled() ? ofyTm().getTransactionTime().toDate() : pojoValue.getTimestamp().toDate(); diff --git a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java index adebb252f..a34960903 100644 --- a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java @@ -25,7 +25,6 @@ import google.registry.batch.DeleteExpiredDomainsAction; import google.registry.batch.DeleteLoadTestDataAction; import google.registry.batch.DeleteProberDataAction; import google.registry.batch.ExpandRecurringBillingEventsAction; -import google.registry.batch.RefreshDnsOnHostRenameAction; import google.registry.batch.RelockDomainAction; import google.registry.batch.ResaveAllEppResourcesAction; import google.registry.batch.ResaveAllEppResourcesPipelineAction; @@ -180,8 +179,6 @@ interface BackendRequestComponent { RefreshDnsAction refreshDnsAction(); - RefreshDnsOnHostRenameAction refreshDnsOnHostRenameAction(); - RelockDomainAction relockDomainAction(); ResaveAllEppResourcesAction resaveAllEppResourcesAction(); diff --git a/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java b/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java index 5558628c4..5b4a0677e 100644 --- a/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java +++ b/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static google.registry.model.ofy.DatastoreTransactionManager.toSqlEntity; import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import static java.util.AbstractMap.SimpleEntry; import static java.util.stream.Collectors.joining; @@ -30,17 +29,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; -import com.googlecode.objectify.Key; import google.registry.model.ImmutableObject; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection; import google.registry.model.index.EppResourceIndex; import google.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex; import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex; import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex; -import google.registry.model.ofy.DatastoreTransactionManager; -import google.registry.model.replay.NonReplicatedEntity; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.JpaRetries; import google.registry.persistence.VKey; import google.registry.util.Clock; @@ -137,15 +130,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { // Postgresql-specific: 'set transaction' command must be called inside a transaction assertInTransaction(); - ReadOnlyCheckingEntityManager entityManager = - (ReadOnlyCheckingEntityManager) getEntityManager(); + EntityManager entityManager = getEntityManager(); // Isolation is hardcoded to REPEATABLE READ, as specified by parent's Javadoc. entityManager .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - .executeUpdateIgnoringReadOnly(); + .executeUpdate(); entityManager .createNativeQuery(String.format("SET TRANSACTION SNAPSHOT '%s'", snapshotId)) - .executeUpdateIgnoringReadOnly(); + .executeUpdate(); return this; } @@ -178,45 +170,12 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { @Override public T transact(Supplier work) { - return transact(work, true); + return retrier.callWithRetry(() -> transactNoRetry(work), JpaRetries::isFailedTxnRetriable); } @Override public T transactWithoutBackup(Supplier work) { - return transact(work, false); - } - - private T transact(Supplier work, boolean withBackup) { - return retrier.callWithRetry( - () -> { - if (inTransaction()) { - return work.get(); - } - TransactionInfo txnInfo = transactionInfo.get(); - txnInfo.entityManager = createReadOnlyCheckingEntityManager(); - EntityTransaction txn = txnInfo.entityManager.getTransaction(); - try { - txn.begin(); - txnInfo.start(clock, withBackup); - T result = work.get(); - txnInfo.recordTransaction(); - txn.commit(); - return result; - } catch (RuntimeException | Error e) { - // Error is unchecked! - try { - txn.rollback(); - logger.atWarning().log("Error during transaction; transaction rolled back."); - } catch (Throwable rollbackException) { - logger.atSevere().withCause(rollbackException).log( - "Rollback failed; suppressing error."); - } - throw e; - } finally { - txnInfo.clear(); - } - }, - JpaRetries::isFailedTxnRetriable); + return transact(work); } @Override @@ -225,13 +184,12 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return work.get(); } TransactionInfo txnInfo = transactionInfo.get(); - txnInfo.entityManager = createReadOnlyCheckingEntityManager(); + txnInfo.entityManager = emf.createEntityManager(); EntityTransaction txn = txnInfo.entityManager.getTransaction(); try { txn.begin(); - txnInfo.start(clock, true); + txnInfo.start(clock); T result = work.get(); - txnInfo.recordTransaction(); txn.commit(); return result; } catch (RuntimeException | Error e) { @@ -320,9 +278,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return; } assertInTransaction(); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - transactionInfo.get().insertObject(toPersist); + transactionInfo.get().insertObject(entity); } @Override @@ -354,9 +310,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return; } assertInTransaction(); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - transactionInfo.get().updateObject(toPersist); + transactionInfo.get().updateObject(entity); } @Override @@ -393,9 +347,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { } assertInTransaction(); checkArgument(exists(entity), "Given entity does not exist"); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - transactionInfo.get().updateObject(toPersist); + transactionInfo.get().updateObject(entity); } @Override @@ -431,7 +383,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { @Override public boolean exists(Object entity) { checkArgumentNotNull(entity, "entity must be specified"); - entity = toSqlEntity(entity); EntityType entityType = getEntityType(entity.getClass()); ImmutableSet entityIds = getEntityIdsFromEntity(entityType, entity); return exists(entityType.getName(), entityIds); @@ -475,7 +426,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { @Override public ImmutableList loadByEntitiesIfPresent(Iterable entities) { return Streams.stream(entities) - .map(DatastoreTransactionManager::toSqlEntity) .filter(this::exists) .map(this::loadByEntity) .collect(toImmutableList()); @@ -510,16 +460,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { public T loadByEntity(T entity) { checkArgumentNotNull(entity, "entity must be specified"); assertInTransaction(); - // If the caller requested a HistoryEntry, load the corresponding *History class - T possibleChild = toSqlEntity(entity); @SuppressWarnings("unchecked") T returnValue = (T) loadByKey( VKey.createSql( - possibleChild.getClass(), + entity.getClass(), // Casting to Serializable is safe according to JPA (JSR 338 sec. 2.4). - (Serializable) emf.getPersistenceUnitUtil().getIdentifier(possibleChild))); + (Serializable) emf.getPersistenceUnitUtil().getIdentifier(entity))); return returnValue; } @@ -570,7 +518,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { String.format("DELETE FROM %s WHERE %s", entityType.getName(), getAndClause(entityIds)); Query query = query(sql); entityIds.forEach(entityId -> query.setParameter(entityId.name, entityId.value)); - transactionInfo.get().addDelete(key); return query.executeUpdate(); } @@ -592,7 +539,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return entity; } assertInTransaction(); - entity = toSqlEntity(entity); T managedEntity = entity; if (!getEntityManager().contains(entity)) { // We don't add the entity to "objectsToSave": once deleted, the object should never be @@ -600,13 +546,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { managedEntity = getEntityManager().merge(entity); } getEntityManager().remove(managedEntity); - - // We check shouldReplicate() in TransactionInfo.addDelete(), but we have to check it here as - // well prior to attempting to create a datastore key because a non-replicated entity may not - // have one. - if (shouldReplicate(entity.getClass())) { - transactionInfo.get().addDelete(VKey.from(Key.create(entity))); - } return managedEntity; } @@ -640,38 +579,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return false; } - @Override - public void putIgnoringReadOnlyWithoutBackup(Object entity) { - checkArgumentNotNull(entity); - if (isEntityOfIgnoredClass(entity)) { - return; - } - assertInTransaction(); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - TransactionInfo txn = transactionInfo.get(); - Object merged = txn.entityManager.mergeIgnoringReadOnly(toPersist); - txn.objectsToSave.add(merged); - txn.addUpdate(toPersist); - } - - @Override - public void deleteIgnoringReadOnlyWithoutBackup(VKey key) { - checkArgumentNotNull(key, "key must be specified"); - assertInTransaction(); - if (IGNORED_ENTITY_CLASSES.contains(key.getKind())) { - return; - } - EntityType entityType = getEntityType(key.getKind()); - ImmutableSet entityIds = getEntityIdsFromSqlKey(entityType, key.getSqlKey()); - String sql = - String.format("DELETE FROM %s WHERE %s", entityType.getName(), getAndClause(entityIds)); - ReadOnlyCheckingQuery query = transactionInfo.get().entityManager.createQuery(sql); - entityIds.forEach(entityId -> query.setParameter(entityId.name, entityId.value)); - transactionInfo.get().addDelete(key); - query.executeUpdateIgnoringReadOnly(); - } - @Override public void assertDelete(VKey key) { if (internalDelete(key) != 1) { @@ -680,10 +587,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { } } - private ReadOnlyCheckingEntityManager createReadOnlyCheckingEntityManager() { - return new ReadOnlyCheckingEntityManager(emf.createEntityManager()); - } - private EntityType getEntityType(Class clazz) { return emf.getMetamodel().entity(clazz); } @@ -820,46 +723,28 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return entity; } - /** Returns true if the entity class should be replicated from SQL to datastore. */ - private static boolean shouldReplicate(Class entityClass) { - return !NonReplicatedEntity.class.isAssignableFrom(entityClass) - && !SqlOnlyEntity.class.isAssignableFrom(entityClass); - } - private static class TransactionInfo { - ReadOnlyCheckingEntityManager entityManager; + EntityManager entityManager; boolean inTransaction = false; DateTime transactionTime; - // Serializable representation of the transaction to be persisted in the Transaction table. - Transaction.Builder contentsBuilder; - // The set of entity objects that have been either persisted (via insert()) or merged (via // put()/update()). If the entity manager returns these as a result of a find() or query // operation, we can not detach them -- detaching removes them from the transaction and causes // them to not be saved to the database -- so we throw an exception instead. - Set objectsToSave = Collections.newSetFromMap(new IdentityHashMap()); + Set objectsToSave = Collections.newSetFromMap(new IdentityHashMap<>()); /** Start a new transaction. */ - private void start(Clock clock, boolean withBackup) { + private void start(Clock clock) { checkArgumentNotNull(clock); inTransaction = true; transactionTime = clock.nowUtc(); - Supplier sqlToDsReplaySupplier = - () -> - DatabaseMigrationStateSchedule.getValueAtTime(transactionTime) - .getReplayDirection() - .equals(ReplayDirection.SQL_TO_DATASTORE); - if (withBackup && sqlToDsReplaySupplier.get()) { - contentsBuilder = new Transaction.Builder(); - } } private void clear() { inTransaction = false; transactionTime = null; - contentsBuilder = null; - objectsToSave = Collections.newSetFromMap(new IdentityHashMap()); + objectsToSave = Collections.newSetFromMap(new IdentityHashMap<>()); if (entityManager != null) { // Close this EntityManager just let the connection pool be able to reuse it, it doesn't // close the underlying database connection. @@ -868,39 +753,16 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { } } - private void addUpdate(Object entity) { - if (contentsBuilder != null && shouldReplicate(entity.getClass())) { - contentsBuilder.addUpdate(entity); - } - } - - private void addDelete(VKey key) { - if (contentsBuilder != null && shouldReplicate(key.getKind())) { - contentsBuilder.addDelete(key); - } - } - - private void recordTransaction() { - if (contentsBuilder != null) { - Transaction persistedTxn = contentsBuilder.build(); - if (!persistedTxn.isEmpty()) { - entityManager.persist(persistedTxn.toEntity()); - } - } - } - /** Does the full "update" on an object including all internal housekeeping. */ private void updateObject(Object object) { Object merged = entityManager.merge(object); objectsToSave.add(merged); - addUpdate(object); } /** Does the full "insert" on a new object including all internal housekeeping. */ private void insertObject(Object object) { entityManager.persist(object); objectsToSave.add(object); - addUpdate(object); } /** Returns true if the object has been persisted/merged and will be saved on commit. */ diff --git a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java b/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java deleted file mode 100644 index d2e929bee..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java +++ /dev/null @@ -1,322 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.List; -import java.util.Map; -import javax.persistence.EntityGraph; -import javax.persistence.EntityManager; -import javax.persistence.EntityManagerFactory; -import javax.persistence.EntityTransaction; -import javax.persistence.FlushModeType; -import javax.persistence.LockModeType; -import javax.persistence.Query; -import javax.persistence.StoredProcedureQuery; -import javax.persistence.TypedQuery; -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaDelete; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.CriteriaUpdate; -import javax.persistence.metamodel.Metamodel; - -/** An {@link EntityManager} that throws exceptions on write actions if in read-only mode. */ -@DeleteAfterMigration -public class ReadOnlyCheckingEntityManager implements EntityManager { - - private final EntityManager delegate; - - public ReadOnlyCheckingEntityManager(EntityManager delegate) { - this.delegate = delegate; - } - - @Override - public void persist(Object entity) { - assertNotReadOnlyMode(); - delegate.persist(entity); - } - - @Override - public T merge(T entity) { - assertNotReadOnlyMode(); - return delegate.merge(entity); - } - - @Override - public void remove(Object entity) { - assertNotReadOnlyMode(); - delegate.remove(entity); - } - - @Override - public T find(Class entityClass, Object primaryKey) { - return delegate.find(entityClass, primaryKey); - } - - @Override - public T find(Class entityClass, Object primaryKey, Map properties) { - return delegate.find(entityClass, primaryKey, properties); - } - - @Override - public T find(Class entityClass, Object primaryKey, LockModeType lockMode) { - return delegate.find(entityClass, primaryKey, lockMode); - } - - @Override - public T find( - Class entityClass, - Object primaryKey, - LockModeType lockMode, - Map properties) { - return delegate.find(entityClass, primaryKey, lockMode, properties); - } - - @Override - public T getReference(Class entityClass, Object primaryKey) { - return delegate.getReference(entityClass, primaryKey); - } - - @Override - public void flush() { - delegate.flush(); - } - - @Override - public void setFlushMode(FlushModeType flushMode) { - delegate.setFlushMode(flushMode); - } - - @Override - public FlushModeType getFlushMode() { - return delegate.getFlushMode(); - } - - @Override - public void lock(Object entity, LockModeType lockMode) { - assertNotReadOnlyMode(); - delegate.lock(entity, lockMode); - } - - @Override - public void lock(Object entity, LockModeType lockMode, Map properties) { - assertNotReadOnlyMode(); - delegate.lock(entity, lockMode, properties); - } - - @Override - public void refresh(Object entity) { - delegate.refresh(entity); - } - - @Override - public void refresh(Object entity, Map properties) { - delegate.refresh(entity, properties); - } - - @Override - public void refresh(Object entity, LockModeType lockMode) { - delegate.refresh(entity, lockMode); - } - - @Override - public void refresh(Object entity, LockModeType lockMode, Map properties) { - delegate.refresh(entity, lockMode, properties); - } - - @Override - public void clear() { - delegate.clear(); - } - - @Override - public void detach(Object entity) { - delegate.detach(entity); - } - - @Override - public boolean contains(Object entity) { - return delegate.contains(entity); - } - - @Override - public LockModeType getLockMode(Object entity) { - return delegate.getLockMode(entity); - } - - @Override - public void setProperty(String propertyName, Object value) { - delegate.setProperty(propertyName, value); - } - - @Override - public Map getProperties() { - return delegate.getProperties(); - } - - @Override - public ReadOnlyCheckingQuery createQuery(String qlString) { - return new ReadOnlyCheckingQuery(delegate.createQuery(qlString)); - } - - @Override - public TypedQuery createQuery(CriteriaQuery criteriaQuery) { - return new ReadOnlyCheckingTypedQuery<>(delegate.createQuery(criteriaQuery)); - } - - @Override - public Query createQuery(CriteriaUpdate updateQuery) { - assertNotReadOnlyMode(); - return delegate.createQuery(updateQuery); - } - - @Override - public Query createQuery(CriteriaDelete deleteQuery) { - assertNotReadOnlyMode(); - return delegate.createQuery(deleteQuery); - } - - @Override - public TypedQuery createQuery(String qlString, Class resultClass) { - return new ReadOnlyCheckingTypedQuery<>(delegate.createQuery(qlString, resultClass)); - } - - @Override - public Query createNamedQuery(String name) { - return new ReadOnlyCheckingQuery(delegate.createNamedQuery(name)); - } - - @Override - public TypedQuery createNamedQuery(String name, Class resultClass) { - return new ReadOnlyCheckingTypedQuery<>(delegate.createNamedQuery(name, resultClass)); - } - - @Override - public ReadOnlyCheckingQuery createNativeQuery(String sqlString) { - return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString)); - } - - @Override - public Query createNativeQuery(String sqlString, Class resultClass) { - return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString, resultClass)); - } - - @Override - public Query createNativeQuery(String sqlString, String resultSetMapping) { - return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString, resultSetMapping)); - } - - @Override - public StoredProcedureQuery createNamedStoredProcedureQuery(String name) { - assertNotReadOnlyMode(); - return delegate.createNamedStoredProcedureQuery(name); - } - - @Override - public StoredProcedureQuery createStoredProcedureQuery(String procedureName) { - assertNotReadOnlyMode(); - return delegate.createStoredProcedureQuery(procedureName); - } - - @Override - public StoredProcedureQuery createStoredProcedureQuery( - String procedureName, Class... resultClasses) { - assertNotReadOnlyMode(); - return delegate.createStoredProcedureQuery(procedureName, resultClasses); - } - - @Override - public StoredProcedureQuery createStoredProcedureQuery( - String procedureName, String... resultSetMappings) { - assertNotReadOnlyMode(); - return delegate.createStoredProcedureQuery(procedureName, resultSetMappings); - } - - @Override - public void joinTransaction() { - delegate.joinTransaction(); - } - - @Override - public boolean isJoinedToTransaction() { - return delegate.isJoinedToTransaction(); - } - - @Override - public T unwrap(Class cls) { - return delegate.unwrap(cls); - } - - @Override - public Object getDelegate() { - return delegate.getDelegate(); - } - - @Override - public void close() { - delegate.close(); - } - - @Override - public boolean isOpen() { - return delegate.isOpen(); - } - - @Override - public EntityTransaction getTransaction() { - return delegate.getTransaction(); - } - - @Override - public EntityManagerFactory getEntityManagerFactory() { - return delegate.getEntityManagerFactory(); - } - - @Override - public CriteriaBuilder getCriteriaBuilder() { - return delegate.getCriteriaBuilder(); - } - - @Override - public Metamodel getMetamodel() { - return delegate.getMetamodel(); - } - - @Override - public EntityGraph createEntityGraph(Class rootType) { - return delegate.createEntityGraph(rootType); - } - - @Override - public EntityGraph createEntityGraph(String graphName) { - return delegate.createEntityGraph(graphName); - } - - @Override - public EntityGraph getEntityGraph(String graphName) { - return delegate.getEntityGraph(graphName); - } - - @Override - public List> getEntityGraphs(Class entityClass) { - return delegate.getEntityGraphs(entityClass); - } - - public T mergeIgnoringReadOnly(T entity) { - return delegate.merge(entity); - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingQuery.java b/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingQuery.java deleted file mode 100644 index 345946253..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingQuery.java +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; -import javax.persistence.FlushModeType; -import javax.persistence.LockModeType; -import javax.persistence.Parameter; -import javax.persistence.Query; -import javax.persistence.TemporalType; - -/** A {@link Query} that throws exceptions on write actions if in read-only mode. */ -@DeleteAfterMigration -class ReadOnlyCheckingQuery implements Query { - - private final Query delegate; - - ReadOnlyCheckingQuery(Query delegate) { - this.delegate = delegate; - } - - @Override - public List getResultList() { - return delegate.getResultList(); - } - - @Override - public Stream getResultStream() { - return delegate.getResultStream(); - } - - @Override - public Object getSingleResult() { - return delegate.getSingleResult(); - } - - @Override - public int executeUpdate() { - assertNotReadOnlyMode(); - return delegate.executeUpdate(); - } - - @Override - public Query setMaxResults(int maxResult) { - return delegate.setMaxResults(maxResult); - } - - @Override - public int getMaxResults() { - return delegate.getMaxResults(); - } - - @Override - public Query setFirstResult(int startPosition) { - return delegate.setFirstResult(startPosition); - } - - @Override - public int getFirstResult() { - return delegate.getFirstResult(); - } - - @Override - public Query setHint(String hintName, Object value) { - return delegate.setHint(hintName, value); - } - - @Override - public Map getHints() { - return delegate.getHints(); - } - - @Override - public Query setParameter(Parameter param, T value) { - return delegate.setParameter(param, value); - } - - @Override - public Query setParameter(Parameter param, Calendar value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public Query setParameter(Parameter param, Date value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public Query setParameter(String name, Object value) { - return delegate.setParameter(name, value); - } - - @Override - public Query setParameter(String name, Calendar value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public Query setParameter(String name, Date value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public Query setParameter(int position, Object value) { - return delegate.setParameter(position, value); - } - - @Override - public Query setParameter(int position, Calendar value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public Query setParameter(int position, Date value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public Set> getParameters() { - return delegate.getParameters(); - } - - @Override - public Parameter getParameter(String name) { - return delegate.getParameter(name); - } - - @Override - public Parameter getParameter(String name, Class type) { - return delegate.getParameter(name, type); - } - - @Override - public Parameter getParameter(int position) { - return delegate.getParameter(position); - } - - @Override - public Parameter getParameter(int position, Class type) { - return delegate.getParameter(position, type); - } - - @Override - public boolean isBound(Parameter param) { - return delegate.isBound(param); - } - - @Override - public T getParameterValue(Parameter param) { - return delegate.getParameterValue(param); - } - - @Override - public Object getParameterValue(String name) { - return delegate.getParameterValue(name); - } - - @Override - public Object getParameterValue(int position) { - return delegate.getParameterValue(position); - } - - @Override - public Query setFlushMode(FlushModeType flushMode) { - return delegate.setFlushMode(flushMode); - } - - @Override - public FlushModeType getFlushMode() { - return delegate.getFlushMode(); - } - - @Override - public Query setLockMode(LockModeType lockMode) { - return delegate.setLockMode(lockMode); - } - - @Override - public LockModeType getLockMode() { - return delegate.getLockMode(); - } - - @Override - public T unwrap(Class cls) { - return delegate.unwrap(cls); - } - - public int executeUpdateIgnoringReadOnly() { - return delegate.executeUpdate(); - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingTypedQuery.java b/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingTypedQuery.java deleted file mode 100644 index e3eab9e2f..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingTypedQuery.java +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; -import javax.persistence.FlushModeType; -import javax.persistence.LockModeType; -import javax.persistence.Parameter; -import javax.persistence.TemporalType; -import javax.persistence.TypedQuery; - -/** A {@link TypedQuery } that throws exceptions on write actions if in read-only mode. */ -@DeleteAfterMigration -class ReadOnlyCheckingTypedQuery implements TypedQuery { - - private final TypedQuery delegate; - - ReadOnlyCheckingTypedQuery(TypedQuery delegate) { - this.delegate = delegate; - } - - @Override - public List getResultList() { - return delegate.getResultList(); - } - - @Override - public Stream getResultStream() { - return delegate.getResultStream(); - } - - @Override - public T getSingleResult() { - return delegate.getSingleResult(); - } - - @Override - public int executeUpdate() { - assertNotReadOnlyMode(); - return delegate.executeUpdate(); - } - - @Override - public TypedQuery setMaxResults(int maxResult) { - return delegate.setMaxResults(maxResult); - } - - @Override - public int getMaxResults() { - return delegate.getMaxResults(); - } - - @Override - public TypedQuery setFirstResult(int startPosition) { - return delegate.setFirstResult(startPosition); - } - - @Override - public int getFirstResult() { - return delegate.getFirstResult(); - } - - @Override - public TypedQuery setHint(String hintName, Object value) { - return delegate.setHint(hintName, value); - } - - @Override - public Map getHints() { - return delegate.getHints(); - } - - @Override - public TypedQuery setParameter(Parameter param, T1 value) { - return delegate.setParameter(param, value); - } - - @Override - public TypedQuery setParameter( - Parameter param, Calendar value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public TypedQuery setParameter(Parameter param, Date value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public TypedQuery setParameter(String name, Object value) { - return delegate.setParameter(name, value); - } - - @Override - public TypedQuery setParameter(String name, Calendar value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public TypedQuery setParameter(String name, Date value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public TypedQuery setParameter(int position, Object value) { - return delegate.setParameter(position, value); - } - - @Override - public TypedQuery setParameter(int position, Calendar value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public TypedQuery setParameter(int position, Date value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public Set> getParameters() { - return delegate.getParameters(); - } - - @Override - public Parameter getParameter(String name) { - return delegate.getParameter(name); - } - - @Override - public Parameter getParameter(String name, Class type) { - return delegate.getParameter(name, type); - } - - @Override - public Parameter getParameter(int position) { - return delegate.getParameter(position); - } - - @Override - public Parameter getParameter(int position, Class type) { - return delegate.getParameter(position, type); - } - - @Override - public boolean isBound(Parameter param) { - return delegate.isBound(param); - } - - @Override - public X getParameterValue(Parameter param) { - return delegate.getParameterValue(param); - } - - @Override - public Object getParameterValue(String name) { - return delegate.getParameterValue(name); - } - - @Override - public Object getParameterValue(int position) { - return delegate.getParameterValue(position); - } - - @Override - public TypedQuery setFlushMode(FlushModeType flushMode) { - return delegate.setFlushMode(flushMode); - } - - @Override - public FlushModeType getFlushMode() { - return delegate.getFlushMode(); - } - - @Override - public TypedQuery setLockMode(LockModeType lockMode) { - return delegate.setLockMode(lockMode); - } - - @Override - public LockModeType getLockMode() { - return delegate.getLockMode(); - } - - @Override - public X unwrap(Class cls) { - return delegate.unwrap(cls); - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/Transaction.java b/core/src/main/java/google/registry/persistence/transaction/Transaction.java deleted file mode 100644 index 75d55a36a..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/Transaction.java +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; - -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityTranslator; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; -import google.registry.model.Buildable; -import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlEntity; -import google.registry.persistence.VKey; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.ObjectStreamClass; - -/** - * A SQL transaction that can be serialized and stored in its own table. - * - *

Transaction is used to store transactions committed to Cloud SQL in a Transaction table during - * the second phase of our migration, during which time we will be asynchronously replaying Cloud - * SQL transactions to datastore. - * - *

TODO(mmuller): Use these from {@link TransactionManager} to store the contents of an SQL - * transaction for asynchronous propagation to datastore. Implement a cron endpoint that reads them - * from the Transaction table and calls writeToDatastore(). - */ -public class Transaction extends ImmutableObject implements Buildable { - - // Version id for persisted objects. Use the creation date for the value, as it's reasonably - // unique and inherently informative. - private static final int VERSION_ID = 20200604; - - // Keep a per-thread flag to keep track of whether we're serializing an entity for a transaction. - // This is used by internal translators to avoid doing things that are dependent on being in a - // datastore transaction and alter the persisted representation of the entity. - private static ThreadLocal inSerializationMode = ThreadLocal.withInitial(() -> false); - - private transient ImmutableList mutations; - - @VisibleForTesting - public ImmutableList getMutations() { - return mutations; - } - - /** Write the entire transaction to the datastore in a datastore transaction. */ - public void writeToDatastore() { - ofyTm() - .transact( - () -> { - for (Mutation mutation : mutations) { - mutation.writeToDatastore(); - } - }); - } - - /** Serialize a transaction to a byte array. */ - public byte[] serialize() { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(baos); - - // Write the transaction version id. This serves as both a version id and a "magic number" to - // protect us against trying to deserialize some random byte array. - out.writeInt(VERSION_ID); - - // Write all of the mutations, preceded by their count. - out.writeInt(mutations.size()); - try { - inSerializationMode.set(true); - for (Mutation mutation : mutations) { - mutation.serializeTo(out); - } - } finally { - inSerializationMode.set(false); - } - - out.close(); - return baos.toByteArray(); - } catch (IOException e) { - throw new IllegalArgumentException(); - } - } - - public static Transaction deserialize(byte[] serializedTransaction) throws IOException { - ObjectInputStream in = - new LenientObjectInputStream(new ByteArrayInputStream(serializedTransaction)); - - // Verify that the data is what we expect. - int version = in.readInt(); - checkArgument( - version == VERSION_ID, "Invalid version id. Expected %s but got %s", VERSION_ID, version); - - Transaction.Builder builder = new Transaction.Builder(); - int mutationCount = in.readInt(); - for (int i = 0; i < mutationCount; ++i) { - try { - builder.add(Mutation.deserializeFrom(in)); - } catch (EOFException e) { - throw new RuntimeException("Serialized transaction terminated prematurely", e); - } - } - if (in.read() != -1) { - throw new RuntimeException("Unread data at the end of a serialized transaction."); - } - return builder.build(); - } - - /** Returns true if the transaction contains no mutations. */ - public boolean isEmpty() { - return mutations.isEmpty(); - } - - /** - * Returns true if we are serializing a transaction in the current thread. - * - *

This should be checked by any Ofy translators prior to making any changes to an entity's - * state representation based on the assumption that we are currently persisting the entity to - * datastore. - */ - public static boolean inSerializationMode() { - return inSerializationMode.get(); - } - - @Override - public Builder asBuilder() { - return new Builder(clone(this)); - } - - public final TransactionEntity toEntity() { - return new TransactionEntity(serialize()); - } - - public static class Builder extends GenericBuilder { - - ImmutableList.Builder listBuilder = new ImmutableList.Builder<>(); - - Builder() {} - - protected Builder(Transaction instance) { - super(instance); - } - - public Builder addUpdate(Object entity) { - checkNotNull(entity); - listBuilder.add(new Update(entity)); - return thisCastToDerived(); - } - - public Builder addDelete(VKey key) { - checkNotNull(key); - listBuilder.add(new Delete(key)); - return thisCastToDerived(); - } - - /** Adds a mutation (mainly intended for serialization) */ - Builder add(Mutation mutation) { - checkNotNull(mutation); - listBuilder.add(mutation); - return thisCastToDerived(); - } - - @Override - public Transaction build() { - getInstance().mutations = listBuilder.build(); - return super.build(); - } - } - - /** Base class for database record mutations. */ - public abstract static class Mutation { - - enum Type { - UPDATE, - DELETE - } - - /** Write the changes in the mutation to the datastore. */ - public abstract void writeToDatastore(); - - /** Serialize the mutation to the output stream. */ - public abstract void serializeTo(ObjectOutputStream out) throws IOException; - - /** Deserialize a mutation from the input stream. */ - public static Mutation deserializeFrom(ObjectInputStream in) throws IOException { - try { - Type type = (Type) in.readObject(); - switch (type) { - case UPDATE: - return Update.deserializeFrom(in); - case DELETE: - return Delete.deserializeFrom(in); - default: - throw new IllegalArgumentException("Unknown enum value: " + type); - } - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - } - - /** - * Record update. - * - *

Note that we don't have to distinguish between add and update, since this is for replay into - * the datastore which makes no such distinction. - * - *

Update serializes its entity using Objectify serialization. - */ - public static class Update extends Mutation { - private Object entity; - - Update(Object entity) { - this.entity = - (entity instanceof SqlEntity) ? ((SqlEntity) entity).toDatastoreEntity().get() : entity; - } - - @Override - public void writeToDatastore() { - ofyTm().putIgnoringReadOnlyWithBackup(entity); - } - - @Override - public void serializeTo(ObjectOutputStream out) throws IOException { - out.writeObject(Type.UPDATE); - Entity realEntity = auditedOfy().toEntity(entity); - EntityProto proto = EntityTranslator.convertToPb(realEntity); - out.write(VERSION_ID); - proto.writeDelimitedTo(out); - } - - @VisibleForTesting - public Object getEntity() { - return entity; - } - - public static Update deserializeFrom(ObjectInputStream in) throws IOException { - EntityProto proto = new EntityProto(); - proto.parseDelimitedFrom(in); - return new Update(auditedOfy().toPojo(EntityTranslator.createFromPb(proto))); - } - } - - /** - * Record deletion. - * - *

Delete serializes its VKey using Java native serialization. - */ - public static class Delete extends Mutation { - private final VKey key; - - Delete(VKey key) { - this.key = key; - } - - @Override - public void writeToDatastore() { - ofyTm().deleteIgnoringReadOnlyWithBackup(key); - } - - @Override - public void serializeTo(ObjectOutputStream out) throws IOException { - out.writeObject(Type.DELETE); - - // Java object serialization works for this. - out.writeObject(key); - } - - @VisibleForTesting - public VKey getKey() { - return key; - } - - public static Delete deserializeFrom(ObjectInputStream in) throws IOException { - try { - return new Delete((VKey) in.readObject()); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - } - - /** - * ObjectInputStream that ignores the UIDs of serialized objects. - * - *

We only really need to deserialize VKeys. However, VKeys have a class object associated with - * them, and if the class is changed and we haven't defined a serialVersionUID for it, we get an - * exception during deserialization. - * - *

It's safe for us to ignore this condition: we only care about attaching the correct local - * class object to the VKey. So this class effectively does so by replacing the class descriptor - * if it's version UID doesn't match that of the local class. - */ - private static class LenientObjectInputStream extends ObjectInputStream { - - public LenientObjectInputStream(InputStream in) throws IOException { - super(in); - } - - @Override - protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFoundException { - ObjectStreamClass persistedDescriptor = super.readClassDescriptor(); - Class localClass = Class.forName(persistedDescriptor.getName()); - ObjectStreamClass localDescriptor = ObjectStreamClass.lookup(localClass); - if (localDescriptor != null) { - if (persistedDescriptor.getSerialVersionUID() != localDescriptor.getSerialVersionUID()) { - return localDescriptor; - } - } - return persistedDescriptor; - } - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java b/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java index af6db5077..4b9022544 100644 --- a/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java +++ b/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java @@ -16,7 +16,7 @@ package google.registry.persistence.transaction; import com.google.common.annotations.VisibleForTesting; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; +import google.registry.model.annotations.DeleteAfterMigration; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; @@ -30,7 +30,8 @@ import javax.persistence.Table; */ @Entity @Table(name = "Transaction") -public class TransactionEntity extends ImmutableObject implements SqlOnlyEntity { +@DeleteAfterMigration +public class TransactionEntity extends ImmutableObject { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java b/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java index b26cadf9d..2cdcc5ed8 100644 --- a/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java +++ b/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java @@ -311,10 +311,4 @@ public interface TransactionManager { /** Returns true if the transaction manager is DatastoreTransactionManager, false otherwise. */ boolean isOfy(); - - /** Performs the write ignoring any read-only restrictions or backup, for use only in replay. */ - void putIgnoringReadOnlyWithoutBackup(Object entity); - - /** Performs the delete ignoring any read-only restrictions or backup, for use only in replay. */ - void deleteIgnoringReadOnlyWithoutBackup(VKey key); } diff --git a/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java b/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java index 215e6bf67..5f643ecc2 100644 --- a/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java +++ b/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java @@ -15,7 +15,6 @@ package google.registry.persistence.transaction; import static com.google.common.base.Preconditions.checkState; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC; import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import com.google.appengine.api.utils.SystemProperty; @@ -23,15 +22,10 @@ import com.google.appengine.api.utils.SystemProperty.Environment.Value; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Suppliers; import google.registry.config.RegistryEnvironment; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.model.ofy.DatastoreTransactionManager; import google.registry.persistence.DaggerPersistenceComponent; import google.registry.tools.RegistryToolEnvironment; -import google.registry.util.Clock; import google.registry.util.NonFinalForTesting; -import google.registry.util.SystemClock; import java.util.Optional; import java.util.function.Supplier; @@ -44,9 +38,6 @@ public final class TransactionManagerFactory { /** Optional override to manually set the transaction manager per-test. */ private static Optional tmForTest = Optional.empty(); - /** The current clock (defined as a variable so we can override it in tests) */ - private static Clock clock = new SystemClock(); - /** Supplier for jpaTm so that it is initialized only once, upon first usage. */ @NonFinalForTesting private static Supplier jpaTm = @@ -56,8 +47,6 @@ public final class TransactionManagerFactory { private static Supplier replicaJpaTm = Suppliers.memoize(TransactionManagerFactory::createReplicaJpaTransactionManager); - private static boolean onBeam = false; - private TransactionManagerFactory() {} private static JpaTransactionManager createJpaTransactionManager() { @@ -102,17 +91,7 @@ public final class TransactionManagerFactory { * the migration schedule or the manually specified per-test transaction manager. */ public static TransactionManager tm() { - if (tmForTest.isPresent()) { - return tmForTest.get(); - } - if (onBeam) { - return jpaTm(); - } - return DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()) - .getPrimaryDatabase() - .equals(PrimaryDatabase.DATASTORE) - ? ofyTm() - : jpaTm(); + return tmForTest.orElseGet(TransactionManagerFactory::jpaTm); } /** @@ -174,7 +153,6 @@ public final class TransactionManagerFactory { public static void setJpaTmOnBeamWorker(Supplier jpaTmSupplier) { checkArgumentNotNull(jpaTmSupplier, "jpaTmSupplier"); jpaTm = Suppliers.memoize(jpaTmSupplier::get); - onBeam = true; } /** @@ -196,39 +174,4 @@ public final class TransactionManagerFactory { public static void removeTmOverrideForTest() { tmForTest = Optional.empty(); } - - public static void assertNotReadOnlyMode() { - if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()).isReadOnly()) { - throw new ReadOnlyModeException(); - } - } - - /** - * Asserts that async actions (contact/host deletes and host renames) are allowed. - * - *

These are allowed at all times except during the {@link - * DatabaseMigrationStateSchedule.MigrationState#DATASTORE_PRIMARY_NO_ASYNC} stage. Note that - * {@link ReadOnlyModeException} may well be thrown during other read-only stages inside the - * transaction manager; this method specifically checks only async actions. - */ - @DeleteAfterMigration - public static void assertAsyncActionsAreAllowed() { - if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()) - .equals(DATASTORE_PRIMARY_NO_ASYNC)) { - throw new ReadOnlyModeException(); - } - } - - /** Allows us to set the clock used by the factory in unit tests. */ - @VisibleForTesting - public static void setClockForTesting(Clock clock) { - TransactionManagerFactory.clock = clock; - } - - /** Registry is currently undergoing maintenance and is in read-only mode. */ - public static class ReadOnlyModeException extends IllegalStateException { - public ReadOnlyModeException() { - super("Registry is currently undergoing maintenance and is in read-only mode"); - } - } } diff --git a/core/src/main/java/google/registry/rde/RdeStagingMapper.java b/core/src/main/java/google/registry/rde/RdeStagingMapper.java index 9a153b398..d1885aa6d 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingMapper.java +++ b/core/src/main/java/google/registry/rde/RdeStagingMapper.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import google.registry.model.EppResource; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DomainBase; import google.registry.model.host.HostResource; @@ -37,6 +38,7 @@ import java.util.function.Supplier; import org.joda.time.DateTime; /** Mapper for {@link RdeStagingAction}. */ +@DeleteAfterMigration public final class RdeStagingMapper extends Mapper { private static final long serialVersionUID = -1518185703789372524L; diff --git a/core/src/main/java/google/registry/rde/RdeStagingReducer.java b/core/src/main/java/google/registry/rde/RdeStagingReducer.java index b959a5677..96573c704 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingReducer.java +++ b/core/src/main/java/google/registry/rde/RdeStagingReducer.java @@ -30,6 +30,7 @@ import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; import google.registry.keyring.api.KeyModule; import google.registry.keyring.api.PgpHelper; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.common.Cursor; import google.registry.model.rde.RdeMode; import google.registry.model.rde.RdeNamingUtils; @@ -59,6 +60,7 @@ import org.joda.time.DateTime; import org.joda.time.Duration; /** Reducer for {@link RdeStagingAction}. */ +@DeleteAfterMigration public final class RdeStagingReducer extends Reducer { private static final long serialVersionUID = 60326234579091203L; diff --git a/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java b/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java index f13d2b721..9d94ede32 100644 --- a/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java +++ b/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java @@ -15,8 +15,6 @@ package google.registry.reporting.billing; import static google.registry.beam.BeamUtils.createJobName; -import static google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase.CLOUD_SQL; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; @@ -31,13 +29,11 @@ import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.persistence.PersistenceModule; import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Action.Service; import google.registry.request.Parameter; -import google.registry.request.RequestParameters; import google.registry.request.Response; import google.registry.request.auth.Auth; import google.registry.util.Clock; @@ -77,7 +73,6 @@ public class GenerateInvoicesAction implements Runnable { private final Clock clock; private final Response response; private final Dataflow dataflow; - private final PrimaryDatabase database; private final CloudTasksUtils cloudTasksUtils; @Inject @@ -88,7 +83,6 @@ public class GenerateInvoicesAction implements Runnable { @Config("billingBucketUrl") String billingBucketUrl, @Config("invoiceFilePrefix") String invoiceFilePrefix, @Parameter(BillingModule.PARAM_SHOULD_PUBLISH) boolean shouldPublish, - @Parameter(RequestParameters.PARAM_DATABASE) PrimaryDatabase database, YearMonth yearMonth, BillingEmailUtils emailUtils, CloudTasksUtils cloudTasksUtils, @@ -98,15 +92,9 @@ public class GenerateInvoicesAction implements Runnable { this.projectId = projectId; this.jobRegion = jobRegion; this.stagingBucketUrl = stagingBucketUrl; - // When generating the invoices using Cloud SQL before database cutover, save the reports in a - // separate bucket so that it does not overwrite the Datastore invoices. - if (tm().isOfy() && database.equals(CLOUD_SQL)) { - billingBucketUrl = billingBucketUrl.concat("-sql"); - } this.billingBucketUrl = billingBucketUrl; this.invoiceFilePrefix = invoiceFilePrefix; this.shouldPublish = shouldPublish; - this.database = database; this.yearMonth = yearMonth; this.emailUtils = emailUtils; this.cloudTasksUtils = cloudTasksUtils; @@ -129,7 +117,6 @@ public class GenerateInvoicesAction implements Runnable { new ImmutableMap.Builder() .put("yearMonth", yearMonth.toString("yyyy-MM")) .put("invoiceFilePrefix", invoiceFilePrefix) - .put("database", database.name()) .put("billingBucketUrl", billingBucketUrl) .put("registryEnvironment", RegistryEnvironment.get().name()) .put( diff --git a/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java b/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java index 8190ec75a..2ef242df5 100644 --- a/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java +++ b/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java @@ -15,7 +15,6 @@ package google.registry.reporting.spec11; import static google.registry.beam.BeamUtils.createJobName; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; @@ -31,12 +30,10 @@ import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; import google.registry.keyring.api.KeyModule.Key; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Action.Service; import google.registry.request.Parameter; -import google.registry.request.RequestParameters; import google.registry.request.Response; import google.registry.request.auth.Auth; import google.registry.util.Clock; @@ -73,7 +70,6 @@ public class GenerateSpec11ReportAction implements Runnable { private final Clock clock; private final Response response; private final Dataflow dataflow; - private final PrimaryDatabase database; private final boolean sendEmail; private final CloudTasksUtils cloudTasksUtils; @@ -85,7 +81,6 @@ public class GenerateSpec11ReportAction implements Runnable { @Config("reportingBucketUrl") String reportingBucketUrl, @Key("safeBrowsingAPIKey") String apiKey, @Parameter(ReportingModule.PARAM_DATE) LocalDate date, - @Parameter(RequestParameters.PARAM_DATABASE) PrimaryDatabase database, @Parameter(ReportingModule.SEND_EMAIL) boolean sendEmail, Clock clock, Response response, @@ -94,13 +89,9 @@ public class GenerateSpec11ReportAction implements Runnable { this.projectId = projectId; this.jobRegion = jobRegion; this.stagingBucketUrl = stagingBucketUrl; - if (tm().isOfy() && database.equals(PrimaryDatabase.CLOUD_SQL)) { - reportingBucketUrl = reportingBucketUrl.concat("-sql"); - } this.reportingBucketUrl = reportingBucketUrl; this.apiKey = apiKey; this.date = date; - this.database = database; this.clock = clock; this.response = response; this.dataflow = dataflow; @@ -121,8 +112,6 @@ public class GenerateSpec11ReportAction implements Runnable { ImmutableMap.of( "safeBrowsingApiKey", apiKey, - "database", - database.name(), ReportingModule.PARAM_DATE, date.toString(), "reportingBucketUrl", diff --git a/core/src/main/java/google/registry/request/RequestModule.java b/core/src/main/java/google/registry/request/RequestModule.java index 1209b6c1c..358c2de23 100644 --- a/core/src/main/java/google/registry/request/RequestModule.java +++ b/core/src/main/java/google/registry/request/RequestModule.java @@ -17,8 +17,6 @@ package google.registry.request; import static com.google.common.net.MediaType.JSON_UTF_8; import static google.registry.model.tld.Registries.assertTldExists; import static google.registry.model.tld.Registries.assertTldsExist; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.request.RequestParameters.extractOptionalParameter; import static google.registry.request.RequestParameters.extractRequiredParameter; import static google.registry.request.RequestParameters.extractSetOfParameters; import static java.nio.charset.StandardCharsets.UTF_8; @@ -33,7 +31,6 @@ import com.google.common.net.MediaType; import com.google.protobuf.ByteString; import dagger.Module; import dagger.Provides; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.UnsupportedMediaTypeException; import google.registry.request.auth.AuthResult; @@ -83,15 +80,6 @@ public final class RequestModule { return tlds; } - @Provides - @Parameter(RequestParameters.PARAM_DATABASE) - static PrimaryDatabase provideDatabase(HttpServletRequest req) { - return extractOptionalParameter(req, RequestParameters.PARAM_DATABASE) - .map(String::toUpperCase) - .map(PrimaryDatabase::valueOf) - .orElse(tm().isOfy() ? PrimaryDatabase.DATASTORE : PrimaryDatabase.CLOUD_SQL); - } - @Provides static Response provideResponse(ResponseImpl response) { return response; diff --git a/core/src/main/java/google/registry/request/RequestParameters.java b/core/src/main/java/google/registry/request/RequestParameters.java index 4c2286f19..a5f2365e0 100644 --- a/core/src/main/java/google/registry/request/RequestParameters.java +++ b/core/src/main/java/google/registry/request/RequestParameters.java @@ -37,14 +37,6 @@ public final class RequestParameters { /** The standardized request parameter name used by any action taking multiple tld parameters. */ public static final String PARAM_TLDS = "tlds"; - /** - * The standardized optional request parameter name used by any action to specify which database - * to use, if the action supports such override. The supported values are (case-insensitive) - * "datastore" and "cloud_sql". - */ - // TODO (jianglai): delete this param after the database migration. - public static final String PARAM_DATABASE = "database"; - /** * Returns first GET or POST parameter associated with {@code name}. * diff --git a/core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java b/core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java deleted file mode 100644 index f9a5a4233..000000000 --- a/core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static com.google.common.base.Preconditions.checkState; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; - -import com.beust.jcommander.Parameters; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.OneTime; -import google.registry.model.domain.DomainBase; -import google.registry.persistence.VKey; - -/** - * Command to dedupe {@link BillingEvent.OneTime} entities having duplicate IDs. - * - *

This command is used to address the duplicate id issue we found for certain {@link - * BillingEvent.OneTime} entities. The command reassigns an application wide unique id to the - * problematic entity and resaves it, it also resaves the entity having reference to the problematic - * entity with the updated id. - * - *

To use this command, you will need to provide the path to a file containing a list of strings - * representing the literal of Objectify key for the problematic entities. An example key literal - * is: - * - * 

- * "DomainBase", "111111-TEST", "HistoryEntry", 2222222, "OneTime", 3333333
- *
- * - *

Note that the double quotes are part of the key literal. The key literal can be retrieved from - * the column __key__.path in BigQuery. - */ -@Parameters( - separators = " =", - commandDescription = "Dedupe BillingEvent.OneTime entities with duplicate IDs.") -public class DedupeOneTimeBillingEventIdsCommand extends ReadEntityFromKeyPathCommand { - - @Override - void process(OneTime entity) { - VKey key = entity.createVKey(); - VKey domainKey = getGrandParentAsDomain(key); - DomainBase domain = tm().transact(() -> tm().loadByKey(domainKey)); - - // The BillingEvent.OneTime entity to be resaved should be the billing event created a few - // years ago, so they should not be referenced from TransferData and GracePeriod in the domain. - assertNotInDomainTransferData(domain, key); - domain - .getGracePeriods() - .forEach( - gracePeriod -> - checkState( - !gracePeriod.getOneTimeBillingEvent().equals(key), - "Entity %s is referenced by a grace period in domain %s", - key, - domainKey)); - - // By setting id to 0L, Buildable.build() will assign an application wide unique id to it. - BillingEvent.OneTime uniqIdBillingEvent = entity.asBuilder().setId(0L).build(); - stageEntityKeyChange(entity, uniqIdBillingEvent); - } - - private static void assertNotInDomainTransferData(DomainBase domainBase, VKey key) { - if (!domainBase.getTransferData().isEmpty()) { - domainBase - .getTransferData() - .getServerApproveEntities() - .forEach( - entityKey -> - checkState( - !entityKey.equals(key), - "Entity %s is referenced by the transfer data in domain %s", - key, - domainBase.createVKey())); - } - } -} diff --git a/core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java b/core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java deleted file mode 100644 index 8a4c680f2..000000000 --- a/core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import com.beust.jcommander.Parameters; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationStateTransition; -import google.registry.model.common.TimedTransitionProperty; - -/** A command to check the current Registry 3.0 migration state of the database. */ -@DeleteAfterMigration -@Parameters(separators = " =", commandDescription = "Check current Registry 3.0 migration state") -public class GetDatabaseMigrationStateCommand implements CommandWithRemoteApi { - - @Override - public void run() throws Exception { - TimedTransitionProperty migrationSchedule = - DatabaseMigrationStateSchedule.get(); - System.out.println( - String.format("Current migration schedule: %s", migrationSchedule.toValueMap())); - } -} diff --git a/core/src/main/java/google/registry/tools/ListTxnsCommand.java b/core/src/main/java/google/registry/tools/ListTxnsCommand.java deleted file mode 100644 index b50f66a7f..000000000 --- a/core/src/main/java/google/registry/tools/ListTxnsCommand.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2022 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.base.Splitter; -import com.google.common.io.BaseEncoding; -import google.registry.model.common.Cursor; -import google.registry.persistence.transaction.Transaction; -import google.registry.persistence.transaction.TransactionEntity; -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; - -/** Lists {@link Cursor} timestamps used by locking rolling cursor tasks, like in RDE. */ -@Parameters(separators = " =", commandDescription = "Lists the contents of the Transaction table.") -final class ListTxnsCommand implements CommandWithRemoteApi { - - @Parameter(names = "--infile", description = "Parse an input file instead of reading from db.") - private String infile; - - @Parameter( - names = "--full_dump", - description = - "Do a full dump of the contents of the transaction. Without this, " - + "just write transactions as CSV lines suitable for ingestion via --infile.") - private boolean fullDump = false; - - @Override - public void run() { - if (infile == null) { - fetchFromDb(); - } else { - parseCsvFile(); - } - } - - private void parseCsvFile() { - try { - BufferedReader src = Files.newBufferedReader(Paths.get(infile), UTF_8); - String line; - while ((line = src.readLine()) != null) { - List cols = Splitter.on(",").splitToList(line); - writeRecord(Integer.parseInt(cols.get(0)), BaseEncoding.base64().decode(cols.get(1))); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void fetchFromDb() { - long lastTransactionId = 0; - List results; - - do { - final long txnId = lastTransactionId; // For use in the lambda. - results = - replicaJpaTm() - .transact( - () -> - replicaJpaTm() - .query( - "select t from TransactionEntity t where id > :lastTransactionId", - TransactionEntity.class) - .setParameter("lastTransactionId", txnId) - .setMaxResults(1000) - .getResultList()); - - for (TransactionEntity txn : results) { - writeRecord(txn.getId(), txn.getContents()); - lastTransactionId = txn.getId(); - } - } while (results.size() > 0); - } - - private void writeRecord(long id, byte[] contents) { - if (fullDump) { - Transaction txn; - try { - txn = Transaction.deserialize(contents); - } catch (IOException ex) { - System.err.printf("Error deserializing transaction %s\n", id); - return; - } - System.out.printf("transaction %s <<<\n", id); - for (Transaction.Mutation mut : txn.getMutations()) { - if (mut instanceof Transaction.Update) { - System.out.println("updating: " + ((Transaction.Update) mut).getEntity()); - } else { - System.out.println("deleting: " + ((Transaction.Delete) mut).getKey()); - } - } - System.out.println(">>>"); - } else { - System.out.printf("%s,%s\n", id, BaseEncoding.base64().encode(contents)); - } - } -} diff --git a/core/src/main/java/google/registry/tools/MutatingCommand.java b/core/src/main/java/google/registry/tools/MutatingCommand.java index d088c6f62..d34be38cd 100644 --- a/core/src/main/java/google/registry/tools/MutatingCommand.java +++ b/core/src/main/java/google/registry/tools/MutatingCommand.java @@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlEntity; import google.registry.persistence.VKey; import java.util.ArrayList; import java.util.HashSet; @@ -90,12 +89,15 @@ public abstract class MutatingCommand extends ConfirmingCommand implements Comma ImmutableObject entity = MoreObjects.firstNonNull(oldEntity, newEntity); // This is one of the few cases where it is acceptable to create an asymmetric VKey (using - // createOfy()). We can use this code on DatastoreOnlyEntity's where we can't construct an + // createOfy()). We can use this code on datastore-only entities where we can't construct a // SQL key. - key = - entity instanceof SqlEntity - ? VKey.from(Key.create(entity)) - : VKey.createOfy(entity.getClass(), Key.create(entity)); + VKey createdKey; + try { + createdKey = VKey.from(Key.create(entity)); + } catch (RuntimeException e) { + createdKey = VKey.createOfy(entity.getClass(), Key.create(entity)); + } + key = createdKey; } /** diff --git a/core/src/main/java/google/registry/tools/RegistryTool.java b/core/src/main/java/google/registry/tools/RegistryTool.java index 81e1513a2..2b865f191 100644 --- a/core/src/main/java/google/registry/tools/RegistryTool.java +++ b/core/src/main/java/google/registry/tools/RegistryTool.java @@ -49,7 +49,6 @@ public final class RegistryTool { .put("create_reserved_list", CreateReservedListCommand.class) .put("create_tld", CreateTldCommand.class) .put("curl", CurlCommand.class) - .put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class) .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class) .put("delete_domain", DeleteDomainCommand.class) .put("delete_host", DeleteHostCommand.class) @@ -67,7 +66,6 @@ public final class RegistryTool { .put("get_allocation_token", GetAllocationTokenCommand.class) .put("get_claims_list", GetClaimsListCommand.class) .put("get_contact", GetContactCommand.class) - .put("get_database_migration_state", GetDatabaseMigrationStateCommand.class) .put("get_domain", GetDomainCommand.class) .put("get_history_entries", GetHistoryEntriesCommand.class) .put("get_host", GetHostCommand.class) @@ -94,7 +92,6 @@ public final class RegistryTool { .put("list_registrars", ListRegistrarsCommand.class) .put("list_reserved_lists", ListReservedListsCommand.class) .put("list_tlds", ListTldsCommand.class) - .put("list_txns", ListTxnsCommand.class) .put("load_snapshot", LoadSnapshotCommand.class) .put("load_test", LoadTestCommand.class) .put("lock_domain", LockDomainCommand.class) @@ -104,13 +101,11 @@ public final class RegistryTool { .put("registrar_contact", RegistrarContactCommand.class) .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class) .put("renew_domain", RenewDomainCommand.class) - .put("replay_txns", ReplayTxnsCommand.class) .put("resave_entities", ResaveEntitiesCommand.class) .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class) .put("resave_epp_resource", ResaveEppResourceCommand.class) .put("save_sql_credential", SaveSqlCredentialCommand.class) .put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class) - .put("set_database_migration_state", SetDatabaseMigrationStateCommand.class) .put("set_num_instances", SetNumInstancesCommand.class) .put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class) .put("setup_ote", SetupOteCommand.class) diff --git a/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java b/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java index c0fabc90f..cbdf04b53 100644 --- a/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java +++ b/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java @@ -16,6 +16,7 @@ package google.registry.tools; import com.beust.jcommander.Parameters; import com.googlecode.objectify.Key; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.domain.DomainBase; import google.registry.persistence.VKey; import javax.annotation.Nullable; @@ -23,6 +24,7 @@ import org.joda.time.DateTime; /** Command to remove the Registry 1.0 key in {@link DomainBase} entity. */ @Parameters(separators = " =", commandDescription = "Remove .") +@DeleteAfterMigration public class RemoveRegistryOneKeyCommand extends ReadEntityFromKeyPathCommand { @Override diff --git a/core/src/main/java/google/registry/tools/ReplayTxnsCommand.java b/core/src/main/java/google/registry/tools/ReplayTxnsCommand.java deleted file mode 100644 index 25f683c0d..000000000 --- a/core/src/main/java/google/registry/tools/ReplayTxnsCommand.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import google.registry.persistence.transaction.Transaction; -import google.registry.persistence.transaction.TransactionEntity; -import java.util.List; - -@Parameters(separators = " =", commandDescription = "Replay a range of transactions.") -public class ReplayTxnsCommand implements CommandWithRemoteApi { - - private static final int BATCH_SIZE = 200; - - @Parameter( - names = {"-s", "--start-txn-id"}, - description = "Transaction id to start replaying at.", - required = true) - long startTxnId; - - @Override - public void run() throws Exception { - List txns; - do { - txns = - replicaJpaTm() - .transact( - () -> - replicaJpaTm() - .query( - "SELECT txn FROM TransactionEntity txn where id >= :startTxn ORDER" - + " BY id", - TransactionEntity.class) - .setParameter("startTxn", startTxnId) - .setMaxResults(BATCH_SIZE) - .getResultList()); - for (TransactionEntity txn : txns) { - System.out.println("Replaying transaction " + txn.getId()); - Transaction.deserialize(txn.getContents()); - startTxnId = txn.getId() + 1; - } - } while (txns.size() > 0); - } -} diff --git a/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java b/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java index 0052492c6..9363b9d41 100644 --- a/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java +++ b/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java @@ -21,6 +21,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import com.beust.jcommander.Parameters; import com.googlecode.objectify.Key; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarContact; import google.registry.model.tld.Registry; @@ -32,6 +33,7 @@ import google.registry.model.tld.Registry; * {@link RegistrarContact}. */ @Parameters(commandDescription = "Re-save all environment entities.") +@DeleteAfterMigration final class ResaveEnvironmentEntitiesCommand implements CommandWithRemoteApi { private static final int BATCH_SIZE = 10; diff --git a/core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java b/core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java deleted file mode 100644 index f880eced6..000000000 --- a/core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.tools.params.TransitionListParameter.MigrationStateTransitions; -import org.joda.time.DateTime; - -/** Command to set the Registry 3.0 database migration state schedule. */ -@DeleteAfterMigration -@Parameters( - separators = " =", - commandDescription = "Set the current database migration state schedule.") -public class SetDatabaseMigrationStateCommand extends ConfirmingCommand - implements CommandWithRemoteApi { - - private static final String WARNING_MESSAGE = - "Attempting to change the schedule with an effect that would take place within the next 10 " - + "minutes. The cache expiration duration is 5 minutes so this MAY BE DANGEROUS.\n"; - - @Parameter( - names = "--migration_schedule", - converter = MigrationStateTransitions.class, - validateWith = MigrationStateTransitions.class, - required = true, - description = - "Comma-delimited list of database transitions, of the form" - + "

A Datastore backup consists of an unsynchronized data export and a sequence of incremental - * Commit Logs that overlap with the export process. Together they can be used to recreate a - * consistent snapshot of the Datastore. - * - *

For convenience of test-writing, the {@link #fakeClock} is advanced by 1 millisecond after - * every transaction is invoked on this store, ensuring strictly increasing timestamps on causally - * dependent transactions. In production, the same ordering is ensured by sleep and retry. - */ -public final class BackupTestStore implements AutoCloseable { - - private static final DateTimeFormatter EXPORT_TIMESTAMP_FORMAT = - DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss_SSS"); - - private final FakeClock fakeClock; - private AppEngineExtension appEngine; - /** For fetching the persisted Datastore Entity directly. */ - private DatastoreService datastoreService; - - private CommitLogCheckpoint prevCommitLogCheckpoint; - - public BackupTestStore(FakeClock fakeClock) throws Exception { - this.fakeClock = fakeClock; - this.appEngine = - new AppEngineExtension.Builder() - .withDatastoreAndCloudSql() - .withoutCannedData() - .withClock(fakeClock) - .build(); - this.appEngine.setUp(); - datastoreService = DatastoreServiceFactory.getDatastoreService(); - } - - /** Returns the timestamp of the transaction. */ - long transact(Iterable deletes, Iterable newOrUpdated) { - long timestamp = fakeClock.nowUtc().getMillis(); - ofyTm() - .transact( - () -> { - auditedOfy().delete().entities(deletes); - auditedOfy().save().entities(newOrUpdated); - }); - fakeClock.advanceOneMilli(); - return timestamp; - } - - /** - * Inserts or updates {@code entities} in the Datastore and returns the timestamp of this - * transaction. - */ - @SafeVarargs - public final long insertOrUpdate(Object... entities) { - long timestamp = fakeClock.nowUtc().getMillis(); - ofyTm().transact(() -> auditedOfy().save().entities(entities).now()); - fakeClock.advanceOneMilli(); - return timestamp; - } - - /** Deletes {@code entities} from the Datastore and returns the timestamp of this transaction. */ - @SafeVarargs - public final long delete(Object... entities) { - long timestamp = fakeClock.nowUtc().getMillis(); - ofyTm().transact(() -> auditedOfy().delete().entities(entities).now()); - fakeClock.advanceOneMilli(); - return timestamp; - } - - /** - * Returns the persisted data that corresponds to {@code ofyEntity} as a Datastore {@link Entity}. - * - *

A typical use case for this method is in a test, when the caller has persisted newly created - * Objectify entity and want to find out the values of certain assign-on-persist properties. See - * {@link VersionedEntity} for more information. - */ - public Entity loadAsDatastoreEntity(Object ofyEntity) { - try { - return datastoreService.get(Key.create(ofyEntity).getRaw()); - } catch (EntityNotFoundException e) { - throw new NoSuchElementException(e.getMessage()); - } - } - - /** - * Returns the persisted data that corresponds to {@code ofyEntity} as an Objectify entity. - * - *

See {@link #loadAsDatastoreEntity} and {@link VersionedEntity} for more information. - */ - public ImmutableObject loadAsOfyEntity(ImmutableObject ofyEntity) { - try { - return auditedOfy().load().fromEntity(datastoreService.get(Key.create(ofyEntity).getRaw())); - } catch (EntityNotFoundException e) { - throw new NoSuchElementException(e.getMessage()); - } - } - /** - * Exports entities of the caller provided types and returns the directory where data is exported. - * - * @param exportRootPath path to the root directory of all exports. A subdirectory will be created - * for this export - * @param pojoTypes java class of all entities to be exported - * @param excludes {@link Set} of {@link Key keys} of the entities not to export.This can be used - * to simulate an inconsistent export - * @return directory where data is exported - */ - File export(String exportRootPath, Iterable> pojoTypes, Set> excludes) - throws IOException { - File exportDirectory = getExportDirectory(exportRootPath); - for (Class pojoType : pojoTypes) { - File perKindFile = - new File( - BackupPaths.getExportFileNameByShard( - exportDirectory.getAbsolutePath(), Key.getKind(pojoType), 0)); - checkState( - perKindFile.getParentFile().mkdirs(), - "Failed to create per-kind export directory for %s.", - perKindFile.getParentFile().getAbsolutePath()); - exportOneKind(perKindFile, pojoType, excludes); - } - return exportDirectory; - } - - private void exportOneKind(File perKindFile, Class pojoType, Set> excludes) - throws IOException { - LevelDbFileBuilder builder = new LevelDbFileBuilder(perKindFile); - for (Object pojo : auditedOfy().load().type(pojoType).iterable()) { - if (!excludes.contains(Key.create(pojo))) { - try { - // Must preserve UpdateTimestamp. Do not use auditedOfy().save().toEntity(pojo)! - builder.addEntity(datastoreService.get(Key.create(pojo).getRaw())); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - builder.build(); - } - - File saveCommitLogs(String commitLogDir) { - CommitLogCheckpoint checkpoint = CommitLogExports.computeCheckpoint(fakeClock); - File commitLogFile = - CommitLogExports.saveCommitLogs(commitLogDir, prevCommitLogCheckpoint, checkpoint); - prevCommitLogCheckpoint = checkpoint; - return commitLogFile; - } - - @Override - public void close() throws Exception { - if (appEngine != null) { - appEngine.tearDown(); - appEngine = null; - } - } - - private File getExportDirectory(String exportRootPath) { - File exportDirectory = - new File(exportRootPath, fakeClock.nowUtc().toString(EXPORT_TIMESTAMP_FORMAT)); - checkState( - exportDirectory.mkdirs(), - "Failed to create export directory %s.", - exportDirectory.getAbsolutePath()); - return exportDirectory; - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java b/core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java deleted file mode 100644 index fac1043ce..000000000 --- a/core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Streams; -import com.googlecode.objectify.Key; -import google.registry.backup.CommitLogImports; -import google.registry.backup.VersionedEntity; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DesignatedContact; -import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.VKey; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import google.registry.tools.LevelDbLogReader; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.stream.Stream; -import org.apache.beam.sdk.values.KV; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** Unit tests for {@link BackupTestStore}. */ -public class BackupTestStoreTest { - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - @TempDir File tempDir; - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension(); - - @RegisterExtension InjectExtension injectExtension = new InjectExtension(); - - private FakeClock fakeClock; - private BackupTestStore store; - - // Test data: - private Registry registry; - private ContactResource contact; - private DomainBase domain; - - @BeforeEach - void beforeEach() throws Exception { - fakeClock = new FakeClock(START_TIME); - store = new BackupTestStore(fakeClock); - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - registry = newRegistry("tld1", "TLD1"); - store.insertOrUpdate(registry); - - contact = newContactResource("contact_1"); - domain = newDomainBase("domain1.tld1", contact); - store.insertOrUpdate(contact, domain); - - // Save persisted data for assertions. - registry = (Registry) store.loadAsOfyEntity(registry); - contact = (ContactResource) store.loadAsOfyEntity(contact); - domain = (DomainBase) store.loadAsOfyEntity(domain); - } - - @AfterEach - void afterEach() throws Exception { - store.close(); - } - - @Test - void export_filesCreated() throws IOException { - String exportRootPath = tempDir.getAbsolutePath(); - assertThat(fakeClock.nowUtc().toString()).isEqualTo("2000-01-01T00:00:00.002Z"); - File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_002"); - assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile()) - .that(exportFolder.exists()) - .isFalse(); - File actualExportFolder = export(exportRootPath, ImmutableSet.of()); - assertThat(actualExportFolder).isEquivalentAccordingToCompareTo(exportFolder); - try (Stream files = - Files.walk(exportFolder.toPath()) - .filter(Files::isRegularFile) - .map(Path::toString) - .map(string -> string.substring(exportFolder.getAbsolutePath().length()))) { - assertThat(files) - .containsExactly( - "/all_namespaces/kind_Registry/output-0", - "/all_namespaces/kind_DomainBase/output-0", - "/all_namespaces/kind_ContactResource/output-0"); - } - } - - @Test - void export_folderNameChangesWithTime() throws IOException { - String exportRootPath = tempDir.getAbsolutePath(); - fakeClock.advanceOneMilli(); - File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_003"); - assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile()) - .that(exportFolder.exists()) - .isFalse(); - assertThat(export(exportRootPath, ImmutableSet.of())) - .isEquivalentAccordingToCompareTo(exportFolder); - } - - @Test - void export_dataReadBack() throws IOException { - String exportRootPath = tempDir.getAbsolutePath(); - File exportFolder = export(exportRootPath, ImmutableSet.of()); - ImmutableList loadedRegistries = - loadExportedEntities(new File(exportFolder, "/all_namespaces/kind_Registry/output-0")); - assertThat(loadedRegistries).containsExactly(registry); - - ImmutableList loadedDomains = - loadExportedEntities(new File(exportFolder, "/all_namespaces/kind_DomainBase/output-0")); - assertThat(loadedDomains).containsExactly(domain); - - ImmutableList loadedContacts = - loadExportedEntities( - new File(exportFolder, "/all_namespaces/kind_ContactResource/output-0")); - assertThat(loadedContacts).containsExactly(contact); - } - - @Test - void export_excludeSomeEntity() throws IOException { - Registry newRegistry = newRegistry("tld2", "TLD2"); - store.insertOrUpdate(newRegistry); - newRegistry = (Registry) store.loadAsOfyEntity(newRegistry); - - String exportRootPath = tempDir.getAbsolutePath(); - File exportFolder = - export( - exportRootPath, ImmutableSet.of(Key.create(getCrossTldKey(), Registry.class, "tld1"))); - ImmutableList loadedRegistries = - loadExportedEntities(new File(exportFolder, "/all_namespaces/kind_Registry/output-0")); - assertThat(loadedRegistries).containsExactly(newRegistry); - } - - @Test - void saveCommitLogs_fileCreated() { - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - assertThat(commitLogFile.exists()).isTrue(); - assertThat(commitLogFile.getName()).isEqualTo("commit_diff_until_2000-01-01T00:00:00.002Z"); - } - - @Test - void saveCommitLogs_inserts() { - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - assertThat(commitLogFile.exists()).isTrue(); - ImmutableList mutations = CommitLogImports.loadEntities(commitLogFile); - InitSqlTestUtils.assertContainsExactlyElementsIn( - mutations, - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(registry)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(contact)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(domain))); - } - - @Test - void saveCommitLogs_deletes() { - fakeClock.advanceOneMilli(); - store.saveCommitLogs(tempDir.getAbsolutePath()); - ContactResource newContact = newContactResource("contact2"); - VKey vKey = newContact.createVKey(); - domain = - domain - .asBuilder() - .setRegistrant(vKey) - .setContacts( - ImmutableSet.of( - DesignatedContact.create(DesignatedContact.Type.ADMIN, vKey), - DesignatedContact.create(DesignatedContact.Type.TECH, vKey))) - .build(); - store.insertOrUpdate(domain, newContact); - store.delete(contact); - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - ImmutableList mutations = CommitLogImports.loadEntities(commitLogFile); - InitSqlTestUtils.assertContainsExactlyElementsIn( - mutations, - KV.of(fakeClock.nowUtc().getMillis() - 1, Key.create(contact).getRaw()), - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(domain)), - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(newContact))); - } - - @Test - void saveCommitLogs_empty() { - fakeClock.advanceOneMilli(); - store.saveCommitLogs(tempDir.getAbsolutePath()); - fakeClock.advanceOneMilli(); - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - assertThat(commitLogFile.exists()).isTrue(); - assertThat(CommitLogImports.loadEntities(commitLogFile)).isEmpty(); - } - - private File export(String exportRootPath, ImmutableSet> excludes) throws IOException { - return store.export( - exportRootPath, - ImmutableList.of(ContactResource.class, DomainBase.class, Registry.class), - excludes); - } - - private static ImmutableList loadExportedEntities(File dataFile) throws IOException { - return Streams.stream(LevelDbLogReader.from(dataFile.toPath())) - .map(InitSqlTestUtils::bytesToEntity) - .map(InitSqlTestUtils::datastoreToOfyEntity) - .collect(ImmutableList.toImmutableList()); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java b/core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java deleted file mode 100644 index 49570ddd3..000000000 --- a/core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import google.registry.backup.VersionedEntity; -import google.registry.beam.TestPipelineExtension; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.nio.file.Files; -import java.nio.file.Path; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** Unit tests for {@link Transforms} related to loading CommitLogs. */ -class CommitLogTransformsTest implements Serializable { - - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - private final FakeClock fakeClock = new FakeClock(START_TIME); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - private transient BackupTestStore store; - private File commitLogsDir; - private File firstCommitLogFile; - - // Canned data: - private transient Registry registry; - private transient ContactResource contact; - private transient DomainBase domain; - - @BeforeEach - void beforeEach() throws Exception { - store = new BackupTestStore(fakeClock); - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - registry = newRegistry("tld1", "TLD1"); - store.insertOrUpdate(registry); - contact = newContactResource("contact_1"); - domain = newDomainBase("domain1.tld1", contact); - store.insertOrUpdate(contact, domain); - - // Save persisted data for assertions. - registry = (Registry) store.loadAsOfyEntity(registry); - contact = (ContactResource) store.loadAsOfyEntity(contact); - domain = (DomainBase) store.loadAsOfyEntity(domain); - - commitLogsDir = Files.createDirectory(tmpDir.resolve("commit_logs")).toFile(); - firstCommitLogFile = store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - } - - @AfterEach - void afterEach() throws Exception { - if (store != null) { - store.close(); - store = null; - } - } - - @Test - void getCommitLogFilePatterns() { - PCollection patterns = - testPipeline.apply( - "Get CommitLog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())); - - ImmutableList expectedPatterns = - ImmutableList.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*"); - - PAssert.that(patterns).containsInAnyOrder(expectedPatterns); - - testPipeline.run(); - } - - @Test - void getFilesByPatterns() { - PCollection fileMetas = - testPipeline - .apply( - "File patterns to metadata", - Create.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*") - .withCoder(StringUtf8Coder.of())) - .apply(Transforms.getFilesByPatterns()); - - // Transform fileMetas to file names for assertions. - PCollection fileNames = - fileMetas.apply( - "File metadata to path string", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement( - @Element Metadata metadata, OutputReceiver out) { - out.output(metadata.resourceId().toString()); - } - })); - - ImmutableList expectedFilenames = - ImmutableList.of(firstCommitLogFile.getAbsolutePath()); - - PAssert.that(fileNames).containsInAnyOrder(expectedFilenames); - - testPipeline.run(); - } - - @Test - void filterCommitLogsByTime() throws IOException { - ImmutableList commitLogFilenames = - ImmutableList.of( - "commit_diff_until_2000-01-01T00:00:00.000Z", - "commit_diff_until_2000-01-01T00:00:00.001Z", - "commit_diff_until_2000-01-01T00:00:00.002Z", - "commit_diff_until_2000-01-01T00:00:00.003Z", - "commit_diff_until_2000-01-01T00:00:00.004Z"); - - for (String name : commitLogFilenames) { - new File(commitLogsDir, name).createNewFile(); - } - - PCollection filteredFilenames = - testPipeline - .apply( - "Get commitlog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())) - .apply("Find commitlog files", Transforms.getFilesByPatterns()) - .apply( - "Filtered by Time", - Transforms.filterCommitLogsByTime( - DateTime.parse("2000-01-01T00:00:00.001Z"), - DateTime.parse("2000-01-01T00:00:00.003Z"))) - .apply( - "Extract path strings", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement( - @Element Metadata fileMeta, OutputReceiver out) { - out.output(fileMeta.resourceId().getFilename()); - } - })); - PAssert.that(filteredFilenames) - .containsInAnyOrder( - "commit_diff_until_2000-01-01T00:00:00.001Z", - "commit_diff_until_2000-01-01T00:00:00.002Z"); - - testPipeline.run(); - } - - @Test - void loadOneCommitLogFile() { - PCollection entities = - testPipeline - .apply( - "Get CommitLog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())) - .apply("Find CommitLogs", Transforms.getFilesByPatterns()) - .apply( - Transforms.loadCommitLogsFromFiles( - ImmutableSet.of("Registry", "ContactResource", "DomainBase"))); - - InitSqlTestUtils.assertContainsExactlyElementsIn( - entities, - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(registry)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(contact)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(domain))); - - testPipeline.run(); - } - - @Test - void loadOneCommitLogFile_filterByKind() { - PCollection entities = - testPipeline - .apply( - "Get CommitLog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())) - .apply("Find CommitLogs", Transforms.getFilesByPatterns()) - .apply( - Transforms.loadCommitLogsFromFiles(ImmutableSet.of("Registry", "ContactResource"))); - - InitSqlTestUtils.assertContainsExactlyElementsIn( - entities, - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(registry)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(contact))); - - testPipeline.run(); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/DatastoreSetupHelper.java b/core/src/test/java/google/registry/beam/initsql/DatastoreSetupHelper.java deleted file mode 100644 index 750a49245..000000000 --- a/core/src/test/java/google/registry/beam/initsql/DatastoreSetupHelper.java +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.model.common.Cursor.CursorType.BRDA; -import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING; -import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE; -import static google.registry.testing.DatabaseHelper.newRegistry; -import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.DatabaseHelper.persistSimpleResource; -import static google.registry.util.DateTimeUtils.END_OF_TIME; -import static google.registry.util.DateTimeUtils.START_OF_TIME; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.flows.domain.DomainFlowUtils; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.Flag; -import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.common.Cursor; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DesignatedContact; -import google.registry.model.domain.DomainAuthInfo; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.DomainHistory; -import google.registry.model.domain.GracePeriod; -import google.registry.model.domain.launch.LaunchNotice; -import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.domain.secdns.DelegationSignerData; -import google.registry.model.domain.token.AllocationToken; -import google.registry.model.eppcommon.AuthInfo.PasswordAuth; -import google.registry.model.eppcommon.StatusValue; -import google.registry.model.eppcommon.Trid; -import google.registry.model.host.HostResource; -import google.registry.model.poll.PollMessage; -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.tld.Registry; -import google.registry.model.transfer.DomainTransferData; -import google.registry.model.transfer.TransferStatus; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.FakeClock; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import org.joda.money.Money; - -/** - * Sets up a test scenario in Datastore. - * - *

The {@link #initializeData} populates Datastore with test data, including {@link DomainBase}, - * {@link DomainHistory}, and commit logs. The up-to-date version of the relevant entities are saved - * in public instance variables (e.g., {@link #domain} for easy access. - */ -public class DatastoreSetupHelper { - - /** - * All kinds of entities to be set up in the Datastore. Must contain all kinds known to {@link - * InitSqlPipeline}. - */ - public static final ImmutableList> ALL_KINDS = - ImmutableList.of( - Registry.class, - Cursor.class, - Registrar.class, - ContactResource.class, - RegistrarContact.class, - DomainBase.class, - HostResource.class, - HistoryEntry.class, - AllocationToken.class, - BillingEvent.Recurring.class, - BillingEvent.OneTime.class, - BillingEvent.Cancellation.class, - PollMessage.class); - - private final Path tmpDir; - private final FakeClock fakeClock; - - public File exportRootDir; - public File exportDir; - public File commitLogDir; - - public Registrar registrar1; - public Registrar registrar2; - public DomainBase domain; - public ContactResource contact1; - public ContactResource contact2; - public HostResource hostResource; - - public DomainHistory historyEntry; - - public Cursor globalCursor; - public Cursor tldCursor; - - public DatastoreSetupHelper(Path tempDir, FakeClock fakeClock) { - this.tmpDir = tempDir; - this.fakeClock = fakeClock; - } - - public DatastoreSetupHelper initializeData() throws Exception { - try (BackupTestStore store = new BackupTestStore(fakeClock)) { - exportRootDir = Files.createDirectory(tmpDir.resolve("exports")).toFile(); - - persistResource(newRegistry("com", "COM")); - registrar1 = persistResource(AppEngineExtension.makeRegistrar1()); - registrar2 = persistResource(AppEngineExtension.makeRegistrar2()); - Key domainKey = Key.create(null, DomainBase.class, "4-COM"); - hostResource = - persistResource( - new HostResource.Builder() - .setHostName("ns1.example.com") - .setSuperordinateDomain(VKey.from(domainKey)) - .setRepoId("1-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setPersistedCurrentSponsorRegistrarId(registrar2.getRegistrarId()) - .build()); - contact1 = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id1") - .setRepoId("2-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setPersistedCurrentSponsorRegistrarId(registrar2.getRegistrarId()) - .build()); - contact2 = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id2") - .setRepoId("3-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setPersistedCurrentSponsorRegistrarId(registrar1.getRegistrarId()) - .build()); - persistSimpleResource( - new RegistrarContact.Builder() - .setParent(registrar1) - .setName("John Abused") - .setEmailAddress("johnabuse@example.com") - .setVisibleInWhoisAsAdmin(true) - .setVisibleInWhoisAsTech(false) - .setPhoneNumber("+1.2125551213") - .setFaxNumber("+1.2125551213") - .setTypes(ImmutableSet.of(RegistrarContact.Type.ABUSE, RegistrarContact.Type.ADMIN)) - .build()); - historyEntry = - persistResource( - new DomainHistory.Builder() - .setDomainRepoId(domainKey.getName()) - .setModificationTime(fakeClock.nowUtc()) - .setRegistrarId(registrar1.getRegistrarId()) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .build()); - persistResource( - new AllocationToken.Builder().setToken("abc123").setTokenType(SINGLE_USE).build()); - Key historyEntryKey = Key.create(historyEntry); - BillingEvent.OneTime onetimeBillEvent = - new BillingEvent.OneTime.Builder() - .setId(1) - .setReason(Reason.RENEW) - .setTargetId("example.com") - .setRegistrarId("TheRegistrar") - .setCost(Money.parse("USD 44.00")) - .setPeriodYears(4) - .setEventTime(fakeClock.nowUtc()) - .setBillingTime(fakeClock.nowUtc()) - .setParent(historyEntryKey) - .build(); - persistResource(onetimeBillEvent); - Key oneTimeBillKey = Key.create(onetimeBillEvent); - BillingEvent.Recurring recurringBillEvent = - new BillingEvent.Recurring.Builder() - .setId(2) - .setReason(Reason.RENEW) - .setFlags(ImmutableSet.of(Flag.AUTO_RENEW)) - .setTargetId("example.com") - .setRegistrarId("TheRegistrar") - .setEventTime(fakeClock.nowUtc()) - .setRecurrenceEndTime(END_OF_TIME) - .setParent(historyEntryKey) - .build(); - persistResource(recurringBillEvent); - VKey recurringBillKey = recurringBillEvent.createVKey(); - PollMessage.Autorenew autorenewPollMessage = - new PollMessage.Autorenew.Builder() - .setId(3L) - .setTargetId("example.com") - .setRegistrarId("TheRegistrar") - .setEventTime(fakeClock.nowUtc()) - .setMsg("Domain was auto-renewed.") - .setParent(historyEntry) - .build(); - persistResource(autorenewPollMessage); - VKey autorenewPollKey = autorenewPollMessage.createVKey(); - PollMessage.OneTime oneTimePollMessage = - new PollMessage.OneTime.Builder() - .setId(1L) - .setParent(historyEntry) - .setEventTime(fakeClock.nowUtc()) - .setRegistrarId("TheRegistrar") - .setMsg(DomainFlowUtils.COLLISION_MESSAGE) - .build(); - persistResource(oneTimePollMessage); - VKey onetimePollKey = oneTimePollMessage.createVKey(); - domain = - persistResource( - new DomainBase.Builder() - .setDomainName("example.com") - .setRepoId("4-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setLastEppUpdateTime(fakeClock.nowUtc()) - .setLastEppUpdateRegistrarId(registrar2.getRegistrarId()) - .setLastTransferTime(fakeClock.nowUtc()) - .setStatusValues( - ImmutableSet.of( - StatusValue.CLIENT_DELETE_PROHIBITED, - StatusValue.SERVER_DELETE_PROHIBITED, - StatusValue.SERVER_TRANSFER_PROHIBITED, - StatusValue.SERVER_UPDATE_PROHIBITED, - StatusValue.SERVER_RENEW_PROHIBITED, - StatusValue.SERVER_HOLD)) - .setRegistrant(contact1.createVKey()) - .setContacts( - ImmutableSet.of( - DesignatedContact.create( - DesignatedContact.Type.ADMIN, contact2.createVKey()))) - .setNameservers(ImmutableSet.of(hostResource.createVKey())) - .setSubordinateHosts(ImmutableSet.of("ns1.example.com")) - .setPersistedCurrentSponsorRegistrarId(registrar2.getRegistrarId()) - .setRegistrationExpirationTime(fakeClock.nowUtc().plusYears(1)) - .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("password"))) - .setDsData( - ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2}))) - .setLaunchNotice( - LaunchNotice.create("tcnid", "validatorId", START_OF_TIME, START_OF_TIME)) - .setTransferData( - new DomainTransferData.Builder() - .setGainingRegistrarId(registrar1.getRegistrarId()) - .setLosingRegistrarId(registrar2.getRegistrarId()) - .setPendingTransferExpirationTime(fakeClock.nowUtc()) - .setServerApproveEntities( - ImmutableSet.of( - VKey.from(oneTimeBillKey), recurringBillKey, autorenewPollKey)) - .setServerApproveBillingEvent(VKey.from(oneTimeBillKey)) - .setServerApproveAutorenewEvent(recurringBillKey) - .setServerApproveAutorenewPollMessage(autorenewPollKey) - .setTransferRequestTime(fakeClock.nowUtc().plusDays(1)) - .setTransferStatus(TransferStatus.SERVER_APPROVED) - .setTransferRequestTrid(Trid.create("client-trid", "server-trid")) - .build()) - .setDeletePollMessage(onetimePollKey) - .setAutorenewBillingEvent(recurringBillKey) - .setAutorenewPollMessage(autorenewPollKey) - .setSmdId("smdid") - .addGracePeriod( - GracePeriod.create( - GracePeriodStatus.ADD, - "4-COM", - fakeClock.nowUtc().plusDays(1), - "TheRegistrar", - null)) - .build()); - persistResource( - new BillingEvent.Cancellation.Builder() - .setReason(Reason.RENEW) - .setTargetId(domain.getDomainName()) - .setRegistrarId(domain.getCurrentSponsorRegistrarId()) - .setEventTime(fakeClock.nowUtc()) - .setBillingTime(fakeClock.nowUtc()) - .setRecurringEventKey(recurringBillEvent.createVKey()) - .setParent(historyEntryKey) - .build()); - globalCursor = persistResource(Cursor.createGlobal(RECURRING_BILLING, fakeClock.nowUtc())); - tldCursor = persistResource(Cursor.create(BRDA, fakeClock.nowUtc(), Registry.get("com"))); - exportDir = store.export(exportRootDir.getAbsolutePath(), ALL_KINDS, ImmutableSet.of()); - commitLogDir = Files.createDirectory(tmpDir.resolve("commits")).toFile(); - fakeClock.advanceOneMilli(); - } - return this; - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java b/core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java deleted file mode 100644 index 6e881729f..000000000 --- a/core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.testing.DatabaseHelper.cloneAndSetAutoTimestamps; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.google.appengine.api.datastore.Entity; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.OneTime; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DesignatedContact; -import google.registry.model.domain.DomainAuthInfo; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.DomainHistory; -import google.registry.model.domain.GracePeriod; -import google.registry.model.domain.launch.LaunchNotice; -import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.domain.secdns.DelegationSignerData; -import google.registry.model.eppcommon.AuthInfo.PasswordAuth; -import google.registry.model.eppcommon.StatusValue; -import google.registry.model.eppcommon.Trid; -import google.registry.model.host.HostResource; -import google.registry.model.ofy.Ofy; -import google.registry.model.poll.PollMessage; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.transfer.DomainTransferData; -import google.registry.model.transfer.TransferStatus; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import org.joda.time.Instant; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Unit tests for {@link DomainBaseUtil}. */ -public class DomainBaseUtilTest { - - private final FakeClock fakeClock = new FakeClock(Instant.now()); - - private DomainBase domain; - private Entity domainEntity; - private Key oneTimeBillKey; - private VKey recurringBillKey; - private Key domainKey; - - @RegisterExtension - AppEngineExtension appEngineExtension = - AppEngineExtension.builder().withDatastoreAndCloudSql().withClock(fakeClock).build(); - - @RegisterExtension InjectExtension injectExtension = new InjectExtension(); - - @BeforeEach - void beforeEach() { - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - createTld("com"); - domainKey = Key.create(null, DomainBase.class, "4-COM"); - VKey hostKey = - persistResource( - new HostResource.Builder() - .setHostName("ns1.example.com") - .setSuperordinateDomain(VKey.from(domainKey)) - .setRepoId("1-COM") - .build()) - .createVKey(); - VKey contact1Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id1") - .setRepoId("2-COM") - .build()) - .createVKey(); - VKey contact2Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id2") - .setRepoId("3-COM") - .build()) - .createVKey(); - Key historyEntryKey = - Key.create( - persistResource( - new DomainHistory.Builder() - .setDomainRepoId(domainKey.getName()) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .setRegistrarId("TheRegistrar") - .setModificationTime(fakeClock.nowUtc().minusYears(1)) - .build())); - oneTimeBillKey = Key.create(historyEntryKey, BillingEvent.OneTime.class, 1); - recurringBillKey = VKey.from(Key.create(historyEntryKey, BillingEvent.Recurring.class, 2)); - VKey autorenewPollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.Autorenew.class, 3)); - VKey onetimePollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.OneTime.class, 1)); - // Set up a new persisted domain entity. - domain = - persistResource( - cloneAndSetAutoTimestamps( - new DomainBase.Builder() - .setDomainName("example.com") - .setRepoId("4-COM") - .setCreationRegistrarId("a registrar") - .setLastEppUpdateTime(fakeClock.nowUtc()) - .setLastEppUpdateRegistrarId("AnotherRegistrar") - .setLastTransferTime(fakeClock.nowUtc()) - .setStatusValues( - ImmutableSet.of( - StatusValue.CLIENT_DELETE_PROHIBITED, - StatusValue.SERVER_DELETE_PROHIBITED, - StatusValue.SERVER_TRANSFER_PROHIBITED, - StatusValue.SERVER_UPDATE_PROHIBITED, - StatusValue.SERVER_RENEW_PROHIBITED, - StatusValue.SERVER_HOLD)) - .setRegistrant(contact1Key) - .setContacts( - ImmutableSet.of( - DesignatedContact.create(DesignatedContact.Type.ADMIN, contact2Key))) - .setNameservers(ImmutableSet.of(hostKey)) - .setSubordinateHosts(ImmutableSet.of("ns1.example.com")) - .setPersistedCurrentSponsorRegistrarId("losing") - .setRegistrationExpirationTime(fakeClock.nowUtc().plusYears(1)) - .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("password"))) - .setDsData( - ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2}))) - .setLaunchNotice( - LaunchNotice.create("tcnid", "validatorId", START_OF_TIME, START_OF_TIME)) - .setTransferData( - new DomainTransferData.Builder() - .setGainingRegistrarId("gaining") - .setLosingRegistrarId("losing") - .setPendingTransferExpirationTime(fakeClock.nowUtc()) - .setServerApproveEntities( - ImmutableSet.of( - VKey.from(oneTimeBillKey), recurringBillKey, autorenewPollKey)) - .setServerApproveBillingEvent(VKey.from(oneTimeBillKey)) - .setServerApproveAutorenewEvent(recurringBillKey) - .setServerApproveAutorenewPollMessage(autorenewPollKey) - .setTransferRequestTime(fakeClock.nowUtc().plusDays(1)) - .setTransferStatus(TransferStatus.SERVER_APPROVED) - .setTransferRequestTrid(Trid.create("client-trid", "server-trid")) - .build()) - .setDeletePollMessage(onetimePollKey) - .setAutorenewBillingEvent(recurringBillKey) - .setAutorenewPollMessage(autorenewPollKey) - .setSmdId("smdid") - .addGracePeriod( - GracePeriod.create( - GracePeriodStatus.ADD, - "4-COM", - fakeClock.nowUtc().plusDays(1), - "registrar", - null)) - .build())); - domainEntity = tm().transact(() -> auditedOfy().toEntity(domain)); - } - - @Test - void removeBillingAndPollAndHosts_allFkeysPresent() { - DomainBase domainTransformedByOfy = - domain - .asBuilder() - .setAutorenewBillingEvent(null) - .setAutorenewPollMessage(null) - .setNameservers(ImmutableSet.of()) - .setDeletePollMessage(null) - .setTransferData(null) - .setGracePeriods(ImmutableSet.of()) - .build(); - DomainBase domainTransformedByUtil = - (DomainBase) auditedOfy().toPojo(DomainBaseUtil.removeBillingAndPollAndHosts(domainEntity)); - // Compensates for the missing INACTIVE status. - domainTransformedByUtil = domainTransformedByUtil.asBuilder().build(); - assertAboutImmutableObjects() - .that(domainTransformedByUtil) - .isEqualExceptFields(domainTransformedByOfy, "revisions", "updateTimestamp"); - } - - @Test - void removeBillingAndPollAndHosts_noFkeysPresent() { - DomainBase domainWithoutFKeys = - domain - .asBuilder() - .setAutorenewBillingEvent(null) - .setAutorenewPollMessage(null) - .setNameservers(ImmutableSet.of()) - .setDeletePollMessage(null) - .setTransferData(null) - .setGracePeriods(ImmutableSet.of()) - .build(); - Entity entityWithoutFkeys = tm().transact(() -> auditedOfy().toEntity(domainWithoutFKeys)); - DomainBase domainTransformedByUtil = - (DomainBase) - auditedOfy().toPojo(DomainBaseUtil.removeBillingAndPollAndHosts(entityWithoutFkeys)); - // Compensates for the missing INACTIVE status. - domainTransformedByUtil = domainTransformedByUtil.asBuilder().build(); - assertAboutImmutableObjects() - .that(domainTransformedByUtil) - .isEqualExceptFields(domainWithoutFKeys, "revisions", "updateTimestamp"); - } - - @Test - void removeBillingAndPollAndHosts_notDomainBase() { - Entity contactEntity = - tm().transact(() -> auditedOfy().toEntity(DatabaseHelper.newContactResource("contact"))); - - assertThrows( - IllegalArgumentException.class, - () -> DomainBaseUtil.removeBillingAndPollAndHosts(contactEntity)); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/ExportLoadingTransformsTest.java b/core/src/test/java/google/registry/beam/initsql/ExportLoadingTransformsTest.java deleted file mode 100644 index ebcad614f..000000000 --- a/core/src/test/java/google/registry/beam/initsql/ExportLoadingTransformsTest.java +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.backup.VersionedEntity; -import google.registry.beam.TestPipelineExtension; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.File; -import java.io.Serializable; -import java.nio.file.Path; -import java.util.Collections; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** - * Unit tests for {@link Transforms} related to loading Datastore exports. - * - *

This class implements {@link Serializable} so that test {@link DoFn} classes may be inlined. - */ -class ExportLoadingTransformsTest implements Serializable { - - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - private static final ImmutableList> ALL_KINDS = - ImmutableList.of(Registry.class, ContactResource.class, DomainBase.class); - private static final ImmutableSet ALL_KIND_STRS = - ALL_KINDS.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet()); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - private FakeClock fakeClock; - private transient BackupTestStore store; - private File exportDir; - - // Canned data: - private transient Registry registry; - private transient ContactResource contact; - private transient DomainBase domain; - - @BeforeEach - void beforeEach() throws Exception { - fakeClock = new FakeClock(START_TIME); - store = new BackupTestStore(fakeClock); - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - registry = newRegistry("tld1", "TLD1"); - store.insertOrUpdate(registry); - - contact = newContactResource("contact_1"); - domain = newDomainBase("domain1.tld1", contact); - store.insertOrUpdate(contact, domain); - - // Save persisted data for assertions. - registry = (Registry) store.loadAsOfyEntity(registry); - contact = (ContactResource) store.loadAsOfyEntity(contact); - domain = (DomainBase) store.loadAsOfyEntity(domain); - - exportDir = store.export(tmpDir.toAbsolutePath().toString(), ALL_KINDS, Collections.EMPTY_SET); - } - - @AfterEach - void afterEach() throws Exception { - if (store != null) { - store.close(); - store = null; - } - } - - @Test - void getExportFilePatterns() { - PCollection patterns = - testPipeline.apply( - "Get Datastore file patterns", - Transforms.getDatastoreExportFilePatterns(exportDir.getAbsolutePath(), ALL_KIND_STRS)); - - ImmutableList expectedPatterns = - ImmutableList.of( - exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/output-*", - exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/output-*", - exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/output-*"); - - PAssert.that(patterns).containsInAnyOrder(expectedPatterns); - - testPipeline.run(); - } - - @Test - void getFilesByPatterns() { - PCollection fileMetas = - testPipeline - .apply( - "File patterns to metadata", - Create.of( - exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/output-*", - exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/output-*", - exportDir.getAbsolutePath() - + "/all_namespaces/kind_ContactResource/output-*") - .withCoder(StringUtf8Coder.of())) - .apply(Transforms.getFilesByPatterns()); - - // Transform fileMetas to file names for assertions. - PCollection fileNames = - fileMetas.apply( - "File metadata to path string", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement( - @Element Metadata metadata, OutputReceiver out) { - out.output(metadata.resourceId().toString()); - } - })); - - ImmutableList expectedFilenames = - ImmutableList.of( - exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/output-0", - exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/output-0", - exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/output-0"); - - PAssert.that(fileNames).containsInAnyOrder(expectedFilenames); - - testPipeline.run(); - } - - @Test - void loadDataFromFiles() { - PCollection entities = - testPipeline - .apply( - "Get Datastore file patterns", - Transforms.getDatastoreExportFilePatterns( - exportDir.getAbsolutePath(), ALL_KIND_STRS)) - .apply("Find Datastore files", Transforms.getFilesByPatterns()) - .apply("Load from Datastore files", Transforms.loadExportDataFromFiles()); - - InitSqlTestUtils.assertContainsExactlyElementsIn( - entities, - KV.of(Transforms.EXPORT_ENTITY_TIME_STAMP, store.loadAsDatastoreEntity(registry)), - KV.of(Transforms.EXPORT_ENTITY_TIME_STAMP, store.loadAsDatastoreEntity(contact)), - KV.of(Transforms.EXPORT_ENTITY_TIME_STAMP, store.loadAsDatastoreEntity(domain))); - - testPipeline.run(); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineGraphTest.java b/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineGraphTest.java deleted file mode 100644 index 7459c6f6a..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineGraphTest.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.truth.TextDiffSubject.assertWithMessageAboutUrlSource; - -import com.google.common.io.Resources; -import google.registry.beam.TestPipelineExtension; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.net.URL; -import org.apache.beam.runners.core.construction.renderer.PipelineDotRenderer; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Manages visualization of {@link InitSqlPipeline}. */ -class InitSqlPipelineGraphTest { - - private static final String GOLDEN_DOT_FILE = "pipeline_golden.dot"; - - private static final String[] OPTIONS_ARGS = - new String[] { - "--commitLogStartTimestamp=2000-01-01TZ", - "--commitLogEndTimestamp=2000-01-02TZ", - "--datastoreExportDir=/somedir", - "--commitLogDir=/someotherdir", - "--registryEnvironment=ALPHA" - }; - - private static final transient InitSqlPipelineOptions options = - PipelineOptionsFactory.fromArgs(OPTIONS_ARGS) - .withValidation() - .as(InitSqlPipelineOptions.class); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(false); - - @Test - void createPipeline_compareGraph() throws IOException { - new InitSqlPipeline(options).setupPipeline(testPipeline); - String dotString = PipelineDotRenderer.toDotString(testPipeline); - URL goldenDotUrl = Resources.getResource(InitSqlPipelineGraphTest.class, GOLDEN_DOT_FILE); - File outputFile = new File(new File(goldenDotUrl.getFile()).getParent(), "pipeline_curr.dot"); - try (PrintStream ps = new PrintStream(outputFile)) { - ps.print(dotString); - } - assertWithMessageAboutUrlSource( - "InitSqlPipeline graph changed. Run :core:updateInitSqlPipelineGraph to update.") - .that(outputFile.toURI().toURL()) - .hasSameContentAs(goldenDotUrl); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineOptionsTest.java b/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineOptionsTest.java deleted file mode 100644 index 299dabdc9..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineOptionsTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.jupiter.api.Test; - -/** Unit tests for {@link google.registry.beam.initsql.InitSqlPipelineOptions}. * */ -public class InitSqlPipelineOptionsTest { - - @Test - void registerToValidate() { - PipelineOptionsFactory.register(InitSqlPipelineOptions.class); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java b/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java deleted file mode 100644 index d08e69304..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects; -import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; - -import google.registry.beam.TestPipelineExtension; -import google.registry.model.common.Cursor; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.host.HostResource; -import google.registry.model.ofy.Ofy; -import google.registry.model.registrar.Registrar; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.nio.file.Path; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.joda.time.DateTime; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** Unit tests for {@link InitSqlPipeline}. */ -class InitSqlPipelineTest { - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - private FakeClock fakeClock = new FakeClock(START_TIME); - - @RegisterExtension - @Order(Order.DEFAULT - 1) - final transient DatastoreEntityExtension datastore = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - @RegisterExtension - final transient JpaIntegrationTestExtension database = - new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationTestExtension(); - - DatastoreSetupHelper setupHelper; - - @BeforeEach - void beforeEach() throws Exception { - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - setupHelper = new DatastoreSetupHelper(tmpDir, fakeClock).initializeData(); - } - - @Test - void runPipeline() { - InitSqlPipelineOptions options = - PipelineOptionsFactory.fromArgs( - "--commitLogStartTimestamp=" + START_TIME, - "--commitLogEndTimestamp=" + fakeClock.nowUtc().plusMillis(1), - "--datastoreExportDir=" + setupHelper.exportDir.getAbsolutePath(), - "--commitLogDir=" + setupHelper.commitLogDir.getAbsolutePath()) - .withValidation() - .as(InitSqlPipelineOptions.class); - InitSqlPipeline initSqlPipeline = new InitSqlPipeline(options); - initSqlPipeline.run(testPipeline).waitUntilFinish(); - assertHostResourceEquals( - jpaTm().transact(() -> jpaTm().loadByKey(setupHelper.hostResource.createVKey())), - setupHelper.hostResource); - assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Registrar.class))) - .comparingElementsUsing(immutableObjectCorrespondence("lastUpdateTime")) - .containsExactly(setupHelper.registrar1, setupHelper.registrar2); - assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(ContactResource.class))) - .comparingElementsUsing(immutableObjectCorrespondence("revisions", "updateTimestamp")) - .containsExactly(setupHelper.contact1, setupHelper.contact2); - assertDomainEquals( - jpaTm().transact(() -> jpaTm().loadByKey(setupHelper.domain.createVKey())), - setupHelper.domain); - assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Cursor.class))) - .comparingElementsUsing(immutableObjectCorrespondence()) - .containsExactly(setupHelper.globalCursor, setupHelper.tldCursor); - } - - private static void assertHostResourceEquals(HostResource actual, HostResource expected) { - assertAboutImmutableObjects() - .that(actual) - .isEqualExceptFields(expected, "superordinateDomain", "revisions", "updateTimestamp"); - assertThat(actual.getSuperordinateDomain().getSqlKey()) - .isEqualTo(expected.getSuperordinateDomain().getSqlKey()); - } - - private static void assertDomainEquals(DomainBase actual, DomainBase expected) { - assertAboutImmutableObjects() - .that(actual) - .isEqualExceptFields( - expected, - "revisions", - "updateTimestamp", - "autorenewPollMessage", - "deletePollMessage", - "nsHosts", - "gracePeriods", - "transferData"); - assertThat(actual.getAdminContact().getSqlKey()) - .isEqualTo(expected.getAdminContact().getSqlKey()); - assertThat(actual.getRegistrant().getSqlKey()).isEqualTo(expected.getRegistrant().getSqlKey()); - assertThat(actual.getNsHosts()).isEqualTo(expected.getNsHosts()); - assertThat(actual.getAutorenewPollMessage().getOfyKey()) - .isEqualTo(expected.getAutorenewPollMessage().getOfyKey()); - assertThat(actual.getDeletePollMessage().getOfyKey()) - .isEqualTo(expected.getDeletePollMessage().getOfyKey()); - assertThat(actual.getUpdateTimestamp()).isEqualTo(expected.getUpdateTimestamp()); - // TODO(weiminyu): check gracePeriods and transferData when it is easier to do - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlTestUtils.java b/core/src/test/java/google/registry/beam/initsql/InitSqlTestUtils.java deleted file mode 100644 index 1aed9d083..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlTestUtils.java +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static org.apache.beam.sdk.values.TypeDescriptors.kvs; -import static org.apache.beam.sdk.values.TypeDescriptors.strings; - -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityTranslator; -import com.google.appengine.api.datastore.Key; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Streams; -import com.google.common.truth.Truth; -import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; -import google.registry.backup.VersionedEntity; -import java.io.Serializable; -import java.util.Collection; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.GroupByKey; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.TypeDescriptor; - -/** Test helpers for populating SQL with Datastore backups. */ -public final class InitSqlTestUtils { - - // Generates unique ids to distinguish reused transforms. - private static final AtomicInteger TRANSFORM_ID_GEN = new AtomicInteger(0); - - /** Converts a Datastore {@link Entity} to an Objectify entity. */ - public static Object datastoreToOfyEntity(Entity entity) { - return auditedOfy().load().fromEntity(entity); - } - - /** Serializes a Datastore {@link Entity} to byte array. */ - public static byte[] entityToBytes(Entity entity) { - return EntityTranslator.convertToPb(entity).toByteArray(); - } - - /** Deserializes raw bytes into {@link Entity}. */ - public static Entity bytesToEntity(byte[] bytes) { - EntityProto proto = new EntityProto(); - proto.parseFrom(bytes); - return EntityTranslator.createFromPb(proto); - } - - /** - * Asserts that the {@code actual} {@link Collection} of {@link VersionedEntity VersionedEntities} - * contains exactly the same elements in the {@code expected} array. - * - *

Each {@code expected} {@link KV key-value pair} refers to a versioned state of an Ofy - * entity. The {@link KV#getKey key} is the timestamp, while the {@link KV#getValue value} is - * either a Datastore {@link Entity} (for an existing entity) or a Datastore {@link Key} (for a - * deleted entity). - * - *

The {@Entity} instances in both actual and expected data are converted to Objectify entities - * so that value-equality checks can be performed. Datastore {@link Entity#equals Entity's equals - * method} only checks key-equality. - */ - @SafeVarargs - public static void assertContainsExactlyElementsIn( - Collection actual, KV... expected) { - assertThat(actual.stream().map(InitSqlTestUtils::rawEntityToOfyWithTimestamp)) - .containsExactlyElementsIn( - Stream.of(expected) - .map(InitSqlTestUtils::expectedToOfyWithTimestamp) - .collect(ImmutableList.toImmutableList())); - } - - /** - * Asserts that the {@code actual} {@link PCollection} of {@link VersionedEntity - * VersionedEntities} contains exactly the same elements in the {@code expected} array. - * - *

This method makes assertions in the pipeline and only use {@link PAssert} on the result. - * This way it supports assertions on Objectify entities, which {@code PAssert} cannot do ( since - * we have not implemented Coders for them). Compared with PAssert-compatible options like {@code - * google.registry.tools.EntityWrapper} or {@link EntityProto}, Objectify entities in Java give - * better-formatted error messages when assertions fail. - * - *

Each {@code expected} {@link KV key-value pair} refers to a versioned state of an Ofy - * entity. The {@link KV#getKey key} is the timestamp, while the {@link KV#getValue value} is - * either a Datastore {@link Entity} (for an existing entity) or a Datastore {@link Key} (for a - * deleted entity). - * - *

The {@Entity} instances in both actual and expected data are converted to Objectify entities - * so that value-equality checks can be performed. Datastore {@link Entity#equals Entity's equals - * method} only checks key-equality. - */ - @SafeVarargs - public static void assertContainsExactlyElementsIn( - PCollection actual, KV... expected) { - PCollection errMsgs = - actual - .apply( - "MapElements_" + TRANSFORM_ID_GEN.getAndIncrement(), - MapElements.into(kvs(strings(), TypeDescriptor.of(VersionedEntity.class))) - .via(rawEntity -> KV.of("The One Key", rawEntity))) - .apply("GroupByKey_" + TRANSFORM_ID_GEN.getAndIncrement(), GroupByKey.create()) - .apply( - "assertContainsExactlyElementsIn_" + TRANSFORM_ID_GEN.getAndIncrement(), - ParDo.of( - new DoFn>, String>() { - @ProcessElement - public void processElement( - @Element KV> input, - OutputReceiver out) { - ImmutableList> actual = - Streams.stream(input.getValue()) - .map(InitSqlTestUtils::rawEntityToOfyWithTimestamp) - .collect(ImmutableList.toImmutableList()); - try { - Truth.assertThat(actual) - .containsExactlyElementsIn( - Stream.of(expected) - .map(InitSqlTestUtils::expectedToOfyWithTimestamp) - .collect(ImmutableList.toImmutableList())); - } catch (AssertionError e) { - out.output(e.toString()); - } - } - })); - PAssert.that(errMsgs).empty(); - } - - private static KV rawEntityToOfyWithTimestamp(VersionedEntity rawEntity) { - return KV.of( - rawEntity.commitTimeMills(), - rawEntity.getEntity().map(InitSqlTestUtils::datastoreToOfyEntity).orElse(rawEntity.key())); - } - - private static KV expectedToOfyWithTimestamp(KV kv) { - return KV.of( - kv.getKey(), - kv.getValue() instanceof Key - ? kv.getValue() - : datastoreToOfyEntity((Entity) kv.getValue())); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/LoadDatastoreSnapshotTest.java b/core/src/test/java/google/registry/beam/initsql/LoadDatastoreSnapshotTest.java deleted file mode 100644 index 1d21b2867..000000000 --- a/core/src/test/java/google/registry/beam/initsql/LoadDatastoreSnapshotTest.java +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.appengine.api.datastore.Entity; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.beam.TestPipelineExtension; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainAuthInfo; -import google.registry.model.domain.DomainBase; -import google.registry.model.eppcommon.AuthInfo.PasswordAuth; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.joda.time.DateTime; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** - * Unit test for {@link Transforms#loadDatastoreSnapshot}. - * - *

The test setup involves three entities, one Registry, one Domain, and two Contacts. Events - * happen in the following order: - * - *

    - *
  1. Registry and a filler Contact are inserted to Datastore. - *
  2. A CommitLog is persisted. - *
  3. Registry is updated. - *
  4. Another Contact and Domain are inserted into Datastore. - *
  5. Datastore is exported, but misses the newly inserted Contact. - *
  6. Filler Contact is deleted. - *
  7. A second CommitLog is persisted. - *
  8. Domain is updated in the Datastore. - *
  9. The third and last CommitLog is persisted. - *
- * - * The final snapshot includes Registry, Domain, and Contact. This scenario verifies that: - * - *
    - *
  • Incremental changes committed before an export does not override the exported valie. - *
  • Entity missed by an export can be recovered from later CommitLogs. - *
  • Multiple changes to an entity is applied in order. - *
  • Deletes are properly handled. - *
- */ -class LoadDatastoreSnapshotTest { - - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - private static final ImmutableList> ALL_KINDS = - ImmutableList.of(Registry.class, ContactResource.class, DomainBase.class); - private static final ImmutableSet ALL_KIND_STRS = - ALL_KINDS.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet()); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - private FakeClock fakeClock; - private File exportRootDir; - private File exportDir; - private File commitLogsDir; - - // Canned data: - private transient Entity dsRegistry; - private transient Entity dsContact; - private transient Entity dsDomain; - - private transient DateTime registryLastUpdateTime; - private transient DateTime contactLastUpdateTime; - private transient DateTime domainLastUpdateTime; - - @BeforeEach - void beforeEach() throws Exception { - fakeClock = new FakeClock(START_TIME); - try (BackupTestStore store = new BackupTestStore(fakeClock)) { - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - exportRootDir = Files.createDirectory(tmpDir.resolve("export_root")).toFile(); - commitLogsDir = Files.createDirectory(tmpDir.resolve("commit_logs")).toFile(); - - Registry registry = newRegistry("tld1", "TLD1"); - ContactResource fillerContact = newContactResource("contact_filler"); - store.insertOrUpdate(registry, fillerContact); - store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - - registry = - registry - .asBuilder() - .setCreateBillingCost(registry.getStandardCreateCost().plus(1.0d)) - .build(); - registryLastUpdateTime = fakeClock.nowUtc(); - store.insertOrUpdate(registry); - - ContactResource contact = newContactResource("contact"); - DomainBase domain = newDomainBase("domain1.tld1", contact); - contactLastUpdateTime = fakeClock.nowUtc(); - store.insertOrUpdate(contact, domain); - exportDir = - store.export( - exportRootDir.getAbsolutePath(), ALL_KINDS, ImmutableSet.of(Key.create(contact))); - - store.delete(fillerContact); - store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - - domain = - domain - .asBuilder() - .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("NewPass"))) - .build(); - domainLastUpdateTime = fakeClock.nowUtc(); - store.insertOrUpdate(domain); - store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - - fakeClock.advanceOneMilli(); - - // Save persisted data for assertions. - dsRegistry = store.loadAsDatastoreEntity(registry); - dsContact = store.loadAsDatastoreEntity(contact); - dsDomain = store.loadAsDatastoreEntity(domain); - } - } - - @Test - void loadDatastoreSnapshot() { - PCollectionTuple snapshot = - testPipeline.apply( - Transforms.loadDatastoreSnapshot( - exportDir.getAbsolutePath(), - commitLogsDir.getAbsolutePath(), - START_TIME, - fakeClock.nowUtc(), - ALL_KIND_STRS)); - InitSqlTestUtils.assertContainsExactlyElementsIn( - snapshot.get(Transforms.createTagForKind("DomainBase")), - KV.of(domainLastUpdateTime.getMillis(), dsDomain)); - InitSqlTestUtils.assertContainsExactlyElementsIn( - snapshot.get(Transforms.createTagForKind("Registry")), - KV.of(registryLastUpdateTime.getMillis(), dsRegistry)); - InitSqlTestUtils.assertContainsExactlyElementsIn( - snapshot.get(Transforms.createTagForKind("ContactResource")), - KV.of(contactLastUpdateTime.getMillis(), dsContact)); - testPipeline.run(); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/TransformsTest.java b/core/src/test/java/google/registry/beam/initsql/TransformsTest.java deleted file mode 100644 index 3d80bdbda..000000000 --- a/core/src/test/java/google/registry/beam/initsql/TransformsTest.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.beam.initsql.Transforms.repairBadData; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newHostResource; - -import com.google.appengine.api.datastore.Entity; -import google.registry.model.domain.DomainBase; -import google.registry.model.host.HostResource; -import google.registry.testing.AppEngineExtension; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Unit tests for {@link Transforms}. */ -public class TransformsTest { - - @RegisterExtension - public final AppEngineExtension appEngine = - AppEngineExtension.builder().withDatastoreAndCloudSql().build(); - - @BeforeEach - void beforeEach() { - createTld("tld"); - } - - @Test - void testRepairBadData_canonicalizesDomainName() { - DomainBase domain = newDomainBase("foobar.tld"); - Entity entity = ofyTm().transact(() -> auditedOfy().toEntity(domain)); - entity.setIndexedProperty("fullyQualifiedDomainName", "FOOBäR.TLD"); - assertThat(((DomainBase) auditedOfy().toPojo(repairBadData(entity))).getDomainName()) - .isEqualTo("xn--foobr-jra.tld"); - } - - @Test - void testRepairBadData_canonicalizesHostName() { - HostResource host = newHostResource("baz.foobar.tld"); - Entity entity = ofyTm().transact(() -> auditedOfy().toEntity(host)); - entity.setIndexedProperty( - "fullyQualifiedHostName", "b̴̹͔͓̣̭̫͇͕̻̬̱͇͗͌́̆̋͒a̶̬̖͚̋̈́̽̇͝͠z̵͠.FOOBäR.TLD"); - assertThat(((HostResource) auditedOfy().toPojo(repairBadData(entity))).getHostName()) - .isEqualTo( - "xn--baz-kdcb2ajgzb4jtg6doej4e6b9am7c7b6c5nd4k7gpa2a9a7dufyewec.xn--foobr-jra.tld"); - } -} diff --git a/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java b/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java index c5c24b9cd..df1905897 100644 --- a/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java +++ b/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java @@ -51,7 +51,6 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; import google.registry.testing.TestDataHelper; -import google.registry.testing.TmOverrideExtension; import google.registry.util.ResourceUtils; import java.io.File; import java.nio.file.Files; @@ -95,10 +94,6 @@ class InvoicingPipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(new FakeClock()).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - @TempDir Path tmpDir; private static final String BILLING_BUCKET_URL = "billing_bucket"; @@ -282,7 +277,6 @@ class InvoicingPipelineTest { @Test void testSuccess_fullSqlPipeline() throws Exception { setupCloudSql(); - options.setDatabase("CLOUD_SQL"); InvoicingPipeline invoicingPipeline = new InvoicingPipeline(options); invoicingPipeline.setupPipeline(pipeline); pipeline.run(options).waitUntilFinish(); diff --git a/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java b/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java index 8371df291..29f7a32d3 100644 --- a/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java +++ b/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java @@ -87,7 +87,6 @@ import google.registry.testing.CloudTasksHelper.TaskMatcher; import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; import google.registry.testing.FakeKeyringModule; -import google.registry.testing.TmOverrideExtension; import java.io.IOException; import java.util.function.Function; import java.util.regex.Matcher; @@ -166,10 +165,6 @@ public class RdePipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - @RegisterExtension final TestPipelineExtension pipeline = TestPipelineExtension.fromOptions(options).enableAbandonedNodeEnforcement(true); diff --git a/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java b/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java index f5f12978a..9714c1909 100644 --- a/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java +++ b/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java @@ -42,7 +42,6 @@ import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.persistence.transaction.TransactionManagerFactory; import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; -import google.registry.testing.TmOverrideExtension; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.joda.time.DateTime; import org.joda.time.Duration; @@ -70,10 +69,6 @@ public class ResaveAllEppResourcesPipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private final ResaveAllEppResourcesPipelineOptions options = PipelineOptionsFactory.create().as(ResaveAllEppResourcesPipelineOptions.class); diff --git a/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java b/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java index 60a8a2dd2..7b3c14d65 100644 --- a/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java +++ b/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java @@ -50,7 +50,6 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; import google.registry.testing.FakeSleeper; -import google.registry.testing.TmOverrideExtension; import google.registry.util.ResourceUtils; import google.registry.util.Retrier; import java.io.File; @@ -128,10 +127,6 @@ class Spec11PipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(new FakeClock()).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private final Spec11PipelineOptions options = PipelineOptionsFactory.create().as(Spec11PipelineOptions.class); @@ -146,7 +141,6 @@ class Spec11PipelineTest { options.setDate(DATE); options.setSafeBrowsingApiKey(SAFE_BROWSING_API_KEY); options.setReportingBucketUrl(reportingBucketUrl.getAbsolutePath()); - options.setDatabase("DATASTORE"); threatMatches = pipeline.apply( Create.of( @@ -199,7 +193,6 @@ class Spec11PipelineTest { @Test void testSuccess_fullSqlPipeline() throws Exception { setupCloudSql(); - options.setDatabase("CLOUD_SQL"); EvaluateSafeBrowsingFn safeBrowsingFn = new EvaluateSafeBrowsingFn( SAFE_BROWSING_API_KEY, diff --git a/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java b/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java index 462c15f28..d90c119b4 100644 --- a/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java +++ b/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java @@ -25,7 +25,6 @@ import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptio import static org.junit.jupiter.api.Assertions.assertThrows; import google.registry.flows.EppException; -import google.registry.flows.EppException.ReadOnlyModeEppException; import google.registry.flows.FlowUtils.NotLoggedInException; import google.registry.flows.ResourceFlowTestCase; import google.registry.flows.contact.ContactFlowUtils.BadInternationalizedPostalInfoException; @@ -33,10 +32,8 @@ import google.registry.flows.contact.ContactFlowUtils.DeclineContactDisclosureFi import google.registry.flows.exceptions.ResourceAlreadyExistsForThisClientException; import google.registry.flows.exceptions.ResourceCreateContentionException; import google.registry.model.contact.ContactResource; -import google.registry.testing.DatabaseHelper; import google.registry.testing.DualDatabaseTest; import google.registry.testing.TestOfyAndSql; -import google.registry.testing.TestOfyOnly; import org.joda.time.DateTime; /** Unit tests for {@link ContactCreateFlow}. */ @@ -137,12 +134,4 @@ class ContactCreateFlowTest extends ResourceFlowTestCase doFailingTest("domain_transfer_request.xml")); - assertAboutEppExceptions().that(thrown).marshalsToXml(); - DatabaseHelper.removeDatabaseMigrationSchedule(); - } } diff --git a/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java b/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java index 18904f7a6..e50f79e87 100644 --- a/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java +++ b/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java @@ -59,7 +59,6 @@ import com.google.common.collect.ImmutableSortedMap; import com.googlecode.objectify.Key; import google.registry.config.RegistryConfig; import google.registry.flows.EppException; -import google.registry.flows.EppException.ReadOnlyModeEppException; import google.registry.flows.EppException.UnimplementedExtensionException; import google.registry.flows.EppRequestSource; import google.registry.flows.FlowUtils.NotLoggedInException; @@ -106,10 +105,8 @@ import google.registry.model.poll.PendingActionNotificationResponse.DomainPendin import google.registry.model.poll.PollMessage; import google.registry.model.tld.Registry; import google.registry.persistence.VKey; -import google.registry.testing.DatabaseHelper; import google.registry.testing.DualDatabaseTest; import google.registry.testing.TestOfyAndSql; -import google.registry.testing.TestOfyOnly; import java.util.Optional; import org.joda.money.Money; import org.joda.time.DateTime; @@ -1745,14 +1742,4 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase { @Test void testFailure_unknownRegistrar() { - deleteResource(getRegistrarBuilder().build()); + registrar.getContacts().forEach(DatabaseHelper::deleteResource); + deleteResource(registrar); doFailingTest("login_valid.xml", BadRegistrarIdException.class); } diff --git a/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java b/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java index c8719812b..958b85e1c 100644 --- a/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java +++ b/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java @@ -42,6 +42,7 @@ import google.registry.model.domain.DomainHistory; import google.registry.model.index.EppResourceIndex; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -52,6 +53,7 @@ import java.util.Set; import org.joda.money.Money; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -60,6 +62,10 @@ class ChildEntityInputTest { private static final DateTime now = DateTime.now(DateTimeZone.UTC); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java b/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java index b28aa8083..a906934d8 100644 --- a/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java +++ b/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java @@ -24,12 +24,14 @@ import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogManifest; import google.registry.testing.AppEngineExtension; import google.registry.testing.DatabaseHelper; +import google.registry.testing.TmOverrideExtension; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; import org.joda.time.DateTime; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -44,6 +46,10 @@ final class CommitLogManifestInputTest { private static final DateTime DATE_TIME_NEW = DateTime.parse("2016-12-19T12:01Z"); private static final DateTime DATE_TIME_NEW2 = DateTime.parse("2017-12-19T12:00Z"); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java b/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java index ed2bf8e1d..d64ccc200 100644 --- a/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java +++ b/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java @@ -37,6 +37,7 @@ import google.registry.model.domain.DomainBase; import google.registry.model.host.HostResource; import google.registry.model.index.EppResourceIndex; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -44,6 +45,7 @@ import java.io.ObjectOutputStream; import java.util.HashSet; import java.util.NoSuchElementException; import java.util.Set; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -52,6 +54,10 @@ class EppResourceInputsTest { private static final double EPSILON = 0.0001; + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java b/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java index bf87fd826..663ec6894 100644 --- a/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java +++ b/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java @@ -22,7 +22,6 @@ import static org.joda.time.DateTimeZone.UTC; import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Ignore; import google.registry.model.common.CrossTldSingleton; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; import google.registry.testing.DualDatabaseTest; import google.registry.testing.TestOfyAndSql; @@ -43,7 +42,6 @@ public class CreateAutoTimestampTest { /** Timestamped class. */ @Entity(name = "CatTestEntity") - @EntityForTesting @javax.persistence.Entity public static class CreateAutoTimestampTestObject extends CrossTldSingleton { @Ignore @javax.persistence.Id long id = SINGLETON_ID; diff --git a/core/src/test/java/google/registry/model/ImmutableObjectTest.java b/core/src/test/java/google/registry/model/ImmutableObjectTest.java index a84905391..5c1c02ae3 100644 --- a/core/src/test/java/google/registry/model/ImmutableObjectTest.java +++ b/core/src/test/java/google/registry/model/ImmutableObjectTest.java @@ -19,7 +19,6 @@ import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Sets.newHashSet; import static com.google.common.truth.Truth.assertThat; import static google.registry.model.ImmutableObject.cloneEmptyToNull; -import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.util.DateTimeUtils.START_OF_TIME; import com.google.common.collect.ImmutableList; @@ -29,7 +28,6 @@ import com.google.common.collect.Iterables; import com.googlecode.objectify.Key; import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Id; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; import google.registry.util.CidrAddressBlock; import java.lang.reflect.Field; @@ -51,6 +49,7 @@ public class ImmutableObjectTest { public final AppEngineExtension appEngine = AppEngineExtension.builder() .withDatastoreAndCloudSql() + .withJpaUnitTestEntities(ValueObject.class) .withOfyTestEntities(ValueObject.class) .build(); @@ -279,10 +278,9 @@ public class ImmutableObjectTest { /** Simple subclass of ImmutableObject. */ @Entity - @EntityForTesting + @javax.persistence.Entity public static class ValueObject extends ImmutableObject { - @Id - long id; + @Id @javax.persistence.Id long id; String value; @@ -294,32 +292,6 @@ public class ImmutableObjectTest { } } - @Test - void testToHydratedString_skipsDoNotHydrate() { - RootObject root = new RootObject(); - root.hydrateMe = Key.create(persistResource(ValueObject.create(1, "foo"))); - root.skipMe = Key.create(persistResource(ValueObject.create(2, "bar"))); - String hydratedString = root.toHydratedString(); - assertThat(hydratedString).contains("foo"); - assertThat(hydratedString).doesNotContain("bar"); - } - - @Test - void testToHydratedString_expandsMaps() { - RootObject root = new RootObject(); - root.map = ImmutableMap.of("foo", Key.create(persistResource(ValueObject.create(1, "bar")))); - String hydratedString = root.toHydratedString(); - assertThat(hydratedString).contains("foo"); - assertThat(hydratedString).contains("bar"); - } - - @Test - void testToHydratedString_expandsCollections() { - RootObject root = new RootObject(); - root.set = ImmutableSet.of(Key.create(persistResource(ValueObject.create(1, "foo")))); - assertThat(root.toHydratedString()).contains("foo"); - } - @Test void testInsignificantFields() { HasInsignificantFields instance1 = diff --git a/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java b/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java index 9b5712305..69be69cc8 100644 --- a/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java +++ b/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java @@ -23,7 +23,6 @@ import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Ignore; import google.registry.model.common.CrossTldSingleton; import google.registry.model.ofy.Ofy; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.VKey; import google.registry.testing.AppEngineExtension; import google.registry.testing.DualDatabaseTest; @@ -59,7 +58,6 @@ public class UpdateAutoTimestampTest { /** Timestamped class. */ @Entity(name = "UatTestEntity") @javax.persistence.Entity - @EntityForTesting public static class UpdateAutoTimestampTestObject extends CrossTldSingleton { @Ignore @javax.persistence.Id long id = SINGLETON_ID; UpdateAutoTimestamp updateTime = UpdateAutoTimestamp.create(null); diff --git a/core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java b/core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java deleted file mode 100644 index 8a0102325..000000000 --- a/core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.common; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_ONLY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_READ_ONLY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_ONLY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY_READ_ONLY; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.EntityTestCase; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.model.domain.token.AllocationToken; -import google.registry.model.domain.token.AllocationToken.TokenType; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; -import google.registry.testing.DatabaseHelper; -import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** Tests for {@link DatabaseMigrationStateSchedule}. */ -public class DatabaseMigrationStateScheduleTest extends EntityTestCase { - - @BeforeEach - void beforeEach() { - fakeClock.setAutoIncrementByOneMilli(); - } - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @Test - void testEmpty_returnsDatastoreOnlyMap() { - assertThat(DatabaseMigrationStateSchedule.getUncached()) - .isEqualTo(DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP); - } - - @Test - void testValidTransitions() { - // First, verify that no-ops are safe - for (MigrationState migrationState : MigrationState.values()) { - runValidTransition(migrationState, migrationState); - } - - // Next, the transitions that will actually cause a change - runValidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY); - - runValidTransition(DATASTORE_PRIMARY, DATASTORE_ONLY); - runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC); - runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY); - - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_ONLY); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_NO_ASYNC); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY); - - runValidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_READ_ONLY); - runValidTransition(SQL_PRIMARY_READ_ONLY, SQL_PRIMARY); - - runValidTransition(SQL_PRIMARY, SQL_PRIMARY_READ_ONLY); - runValidTransition(SQL_PRIMARY, SQL_ONLY); - - runValidTransition(SQL_ONLY, SQL_PRIMARY); - } - - @Test - void testInvalidTransitions() { - runInvalidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY); - runInvalidTransition(DATASTORE_ONLY, SQL_ONLY); - - runInvalidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY); - runInvalidTransition(DATASTORE_PRIMARY, SQL_ONLY); - - runInvalidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_ONLY); - - runInvalidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_ONLY); - runInvalidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_PRIMARY); - runInvalidTransition(SQL_PRIMARY_READ_ONLY, SQL_ONLY); - - runInvalidTransition(SQL_PRIMARY, DATASTORE_ONLY); - runInvalidTransition(SQL_PRIMARY, DATASTORE_PRIMARY); - runInvalidTransition(SQL_PRIMARY, DATASTORE_PRIMARY_READ_ONLY); - - runInvalidTransition(SQL_ONLY, DATASTORE_ONLY); - runInvalidTransition(SQL_ONLY, DATASTORE_PRIMARY); - runInvalidTransition(SQL_ONLY, DATASTORE_PRIMARY_READ_ONLY); - } - - @Test - void testFailure_newMapImpliesInvalidChangeNow() { - DateTime startTime = fakeClock.nowUtc(); - fakeClock.advanceBy(Duration.standardHours(6)); - - // The new map is valid by itself, but not with the current state of DATASTORE_ONLY because the - // new map implies that the current state is DATASTORE_PRIMARY_READ_ONLY - ImmutableSortedMap nowInvalidMap = - ImmutableSortedMap.naturalOrder() - .put(START_OF_TIME, DATASTORE_ONLY) - .put(startTime.plusHours(1), DATASTORE_PRIMARY) - .put(startTime.plusHours(2), DATASTORE_PRIMARY_NO_ASYNC) - .put(startTime.plusHours(3), DATASTORE_PRIMARY_READ_ONLY) - .build(); - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(nowInvalidMap))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "Cannot transition from current state-as-of-now DATASTORE_ONLY " - + "to new state-as-of-now DATASTORE_PRIMARY_READ_ONLY"); - } - - @Test - void testFailure_notInTransaction() { - IllegalStateException thrown = - assertThrows( - IllegalStateException.class, - () -> - DatabaseMigrationStateSchedule.set( - DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP.toValueMap())); - assertThat(thrown).hasMessageThat().isEqualTo("Not in a transaction"); - } - - @Test - void testSuccess_factoryUsesSchedule() { - assertThat(tm().isOfy()).isTrue(); - // set the schedule to have converted to SQL_PRIMARY in the past - fakeClock.setTo(START_OF_TIME.plusDays(1)); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY); - assertThat(tm().isOfy()).isFalse(); - } - - @Test - void testSuccess_factoryUsesReadOnly() { - createTld("tld"); - fakeClock.setTo(START_OF_TIME.plusDays(1)); - AllocationToken token = - new AllocationToken.Builder().setToken("token").setTokenType(TokenType.SINGLE_USE).build(); - runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC); - runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY); - assertThrows(ReadOnlyModeException.class, () -> persistResource(token)); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY); - assertThrows(ReadOnlyModeException.class, () -> persistResource(token)); - runValidTransition(SQL_PRIMARY_READ_ONLY, SQL_PRIMARY); - persistResource(token); - } - - private void runValidTransition(MigrationState from, MigrationState to) { - ImmutableSortedMap transitions = - createMapEndingWithTransition(from, to); - jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions)); - assertThat(DatabaseMigrationStateSchedule.getUncached().toValueMap()) - .containsExactlyEntriesIn(transitions); - } - - private void runInvalidTransition(MigrationState from, MigrationState to) { - ImmutableSortedMap transitions = - createMapEndingWithTransition(from, to); - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - String.format("validStateTransitions map cannot transition from %s to %s.", from, to)); - } - - // Create a transition map that is valid up to the "from" transition, then add the "to" transition - private ImmutableSortedMap createMapEndingWithTransition( - MigrationState from, MigrationState to) { - ImmutableSortedMap.Builder builder = - ImmutableSortedMap.naturalOrder(); - builder.put(START_OF_TIME, DATASTORE_ONLY); - MigrationState[] allMigrationStates = MigrationState.values(); - for (int i = 0; i < allMigrationStates.length; i++) { - builder.put(fakeClock.nowUtc().plusMinutes(i), allMigrationStates[i]); - if (allMigrationStates[i].equals(from)) { - break; - } - } - builder.put(fakeClock.nowUtc().plusDays(1), to); - return builder.build(); - } -} diff --git a/core/src/test/java/google/registry/model/domain/DomainBaseTest.java b/core/src/test/java/google/registry/model/domain/DomainBaseTest.java index 936bf8dc6..a92d69a13 100644 --- a/core/src/test/java/google/registry/model/domain/DomainBaseTest.java +++ b/core/src/test/java/google/registry/model/domain/DomainBaseTest.java @@ -23,8 +23,12 @@ import static google.registry.testing.DatabaseHelper.cloneAndSetAutoTimestamps; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.newDomainBase; import static google.registry.testing.DatabaseHelper.newHostResource; +import static google.registry.testing.DatabaseHelper.persistActiveContact; +import static google.registry.testing.DatabaseHelper.persistActiveDomain; +import static google.registry.testing.DatabaseHelper.persistActiveHost; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DomainBaseSubject.assertAboutDomains; +import static google.registry.util.DateTimeUtils.END_OF_TIME; import static google.registry.util.DateTimeUtils.START_OF_TIME; import static org.joda.money.CurrencyUnit.USD; import static org.joda.time.DateTimeZone.UTC; @@ -41,6 +45,7 @@ import google.registry.model.EntityTestCase; import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObjectSubject; import google.registry.model.billing.BillingEvent; +import google.registry.model.billing.BillingEvent.Flag; import google.registry.model.billing.BillingEvent.Reason; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DesignatedContact.Type; @@ -70,50 +75,55 @@ public class DomainBaseTest extends EntityTestCase { private DomainBase domain; private VKey oneTimeBillKey; private VKey recurringBillKey; - private Key historyEntryKey; + private DomainHistory domainHistory; private VKey contact1Key, contact2Key; @BeforeEach void setUp() { createTld("com"); - VKey domainKey = VKey.from(Key.create(null, DomainBase.class, "4-COM")); - VKey hostKey = + domain = persistActiveDomain("example.com"); + VKey hostKey = persistActiveHost("ns1.example.com").createVKey(); + contact1Key = persistActiveContact("contact_id1").createVKey(); + contact2Key = persistActiveContact("contact_id1").createVKey(); + domainHistory = persistResource( - new HostResource.Builder() - .setHostName("ns1.example.com") - .setSuperordinateDomain(domainKey) - .setRepoId("1-COM") + new DomainHistory.Builder() + .setDomainRepoId(domain.createVKey().getOfyKey().getName()) + .setModificationTime(fakeClock.nowUtc()) + .setType(HistoryEntry.Type.DOMAIN_CREATE) + .setRegistrarId("TheRegistrar") + .build()); + oneTimeBillKey = + persistResource( + new BillingEvent.OneTime.Builder() + // Use SERVER_STATUS so we don't have to add a period. + .setReason(Reason.SERVER_STATUS) + .setTargetId(domain.getDomainName()) + .setRegistrarId(domain.getCurrentSponsorRegistrarId()) + .setDomainRepoId(domain.getRepoId()) + .setBillingTime(DateTime.now(UTC)) + .setCost(Money.of(USD, 100)) + .setEventTime(DateTime.now(UTC).plusYears(1)) + .setParent(domainHistory) .build()) .createVKey(); - contact1Key = + recurringBillKey = persistResource( - new ContactResource.Builder() - .setContactId("contact_id1") - .setRepoId("2-COM") + new BillingEvent.Recurring.Builder() + .setReason(Reason.RENEW) + .setFlags(ImmutableSet.of(Flag.AUTO_RENEW)) + .setTargetId(domain.getDomainName()) + .setRegistrarId(domain.getCurrentSponsorRegistrarId()) + .setDomainRepoId(domain.getRepoId()) + .setEventTime(DateTime.now(UTC).plusYears(1)) + .setRecurrenceEndTime(END_OF_TIME) + .setParent(domainHistory) .build()) .createVKey(); - contact2Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id2") - .setRepoId("3-COM") - .build()) - .createVKey(); - historyEntryKey = - Key.create( - persistResource( - new DomainHistory.Builder() - .setDomainRepoId(domainKey.getOfyKey().getName()) - .setModificationTime(fakeClock.nowUtc()) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .setRegistrarId("aregistrar") - .build())); - oneTimeBillKey = VKey.from(Key.create(historyEntryKey, BillingEvent.OneTime.class, 1)); - recurringBillKey = VKey.from(Key.create(historyEntryKey, BillingEvent.Recurring.class, 2)); VKey autorenewPollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.Autorenew.class, 3)); + VKey.from(Key.create(Key.create(domainHistory), PollMessage.Autorenew.class, 3)); VKey onetimePollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.OneTime.class, 1)); + VKey.from(Key.create(Key.create(domainHistory), PollMessage.OneTime.class, 1)); // Set up a new persisted domain entity. domain = persistResource( @@ -121,9 +131,9 @@ public class DomainBaseTest extends EntityTestCase { new DomainBase.Builder() .setDomainName("example.com") .setRepoId("4-COM") - .setCreationRegistrarId("aregistrar") + .setCreationRegistrarId("TheRegistrar") .setLastEppUpdateTime(fakeClock.nowUtc()) - .setLastEppUpdateRegistrarId("AnotherRegistrar") + .setLastEppUpdateRegistrarId("NewRegistrar") .setLastTransferTime(fakeClock.nowUtc()) .setStatusValues( ImmutableSet.of( @@ -137,7 +147,7 @@ public class DomainBaseTest extends EntityTestCase { .setContacts(ImmutableSet.of(DesignatedContact.create(Type.ADMIN, contact2Key))) .setNameservers(ImmutableSet.of(hostKey)) .setSubordinateHosts(ImmutableSet.of("ns1.example.com")) - .setPersistedCurrentSponsorRegistrarId("losing") + .setPersistedCurrentSponsorRegistrarId("NewRegistrar") .setRegistrationExpirationTime(fakeClock.nowUtc().plusYears(1)) .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("password"))) .setDsData( @@ -146,8 +156,8 @@ public class DomainBaseTest extends EntityTestCase { LaunchNotice.create("tcnid", "validatorId", START_OF_TIME, START_OF_TIME)) .setTransferData( new DomainTransferData.Builder() - .setGainingRegistrarId("gaining") - .setLosingRegistrarId("losing") + .setGainingRegistrarId("TheRegistrar") + .setLosingRegistrarId("NewRegistrar") .setPendingTransferExpirationTime(fakeClock.nowUtc()) .setServerApproveEntities( ImmutableSet.of(oneTimeBillKey, recurringBillKey, autorenewPollKey)) @@ -165,10 +175,10 @@ public class DomainBaseTest extends EntityTestCase { .addGracePeriod( GracePeriod.create( GracePeriodStatus.ADD, - "4-COM", + domain.getRepoId(), fakeClock.nowUtc().plusDays(1), - "registrar", - null)) + "TheRegistrar", + oneTimeBillKey)) .setAutorenewEndTime(Optional.of(fakeClock.nowUtc().plusYears(2))) .setDnsRefreshRequestTime(Optional.of(fakeClock.nowUtc())) .build())); @@ -192,28 +202,15 @@ public class DomainBaseTest extends EntityTestCase { @Test void testVKeyRestoration() { - assertThat(domain.deletePollMessageHistoryId).isEqualTo(historyEntryKey.getId()); - assertThat(domain.autorenewBillingEventHistoryId).isEqualTo(historyEntryKey.getId()); - assertThat(domain.autorenewPollMessageHistoryId).isEqualTo(historyEntryKey.getId()); + assertThat(domain.deletePollMessageHistoryId).isEqualTo(domainHistory.getId()); + assertThat(domain.autorenewBillingEventHistoryId).isEqualTo(domainHistory.getId()); + assertThat(domain.autorenewPollMessageHistoryId).isEqualTo(domainHistory.getId()); assertThat(domain.getTransferData().getServerApproveBillingEventHistoryId()) - .isEqualTo(historyEntryKey.getId()); + .isEqualTo(domainHistory.getId()); assertThat(domain.getTransferData().getServerApproveAutorenewEventHistoryId()) - .isEqualTo(historyEntryKey.getId()); + .isEqualTo(domainHistory.getId()); assertThat(domain.getTransferData().getServerApproveAutorenewPollMessageHistoryId()) - .isEqualTo(historyEntryKey.getId()); - } - - @Test - void testIndexing() throws Exception { - verifyDatastoreIndexing( - domain, - "allContacts.contact", - "fullyQualifiedDomainName", - "nsHosts", - "currentSponsorClientId", - "deletionTime", - "tld", - "autorenewEndTime"); + .isEqualTo(domainHistory.getId()); } @Test @@ -366,7 +363,7 @@ public class DomainBaseTest extends EntityTestCase { VKey newAutorenewEvent) { assertThat(domain.getTransferData().getTransferStatus()) .isEqualTo(TransferStatus.SERVER_APPROVED); - assertThat(domain.getCurrentSponsorRegistrarId()).isEqualTo("winner"); + assertThat(domain.getCurrentSponsorRegistrarId()).isEqualTo("TheRegistrar"); assertThat(domain.getLastTransferTime()).isEqualTo(fakeClock.nowUtc().plusDays(1)); assertThat(domain.getRegistrationExpirationTime()).isEqualTo(newExpirationTime); assertThat(domain.getAutorenewBillingEvent()).isEqualTo(newAutorenewEvent); @@ -374,18 +371,19 @@ public class DomainBaseTest extends EntityTestCase { private void doExpiredTransferTest(DateTime oldExpirationTime) { DomainHistory historyEntry = - new DomainHistory.Builder() - .setDomain(domain) - .setModificationTime(fakeClock.nowUtc()) - .setRegistrarId(domain.getCurrentSponsorRegistrarId()) - .setType(HistoryEntry.Type.DOMAIN_TRANSFER_REQUEST) - .build(); + persistResource( + new DomainHistory.Builder() + .setDomain(domain) + .setModificationTime(fakeClock.nowUtc()) + .setRegistrarId(domain.getCurrentSponsorRegistrarId()) + .setType(HistoryEntry.Type.DOMAIN_TRANSFER_REQUEST) + .build()); BillingEvent.OneTime transferBillingEvent = persistResource( new BillingEvent.OneTime.Builder() .setReason(Reason.TRANSFER) - .setRegistrarId("winner") - .setTargetId("example.com") + .setRegistrarId("TheRegistrar") + .setTargetId(domain.getDomainName()) .setEventTime(fakeClock.nowUtc()) .setBillingTime( fakeClock @@ -407,7 +405,7 @@ public class DomainBaseTest extends EntityTestCase { .setTransferStatus(TransferStatus.PENDING) .setTransferRequestTime(fakeClock.nowUtc().minusDays(4)) .setPendingTransferExpirationTime(fakeClock.nowUtc().plusDays(1)) - .setGainingRegistrarId("winner") + .setGainingRegistrarId("TheRegistrar") .setServerApproveBillingEvent(transferBillingEvent.createVKey()) .setServerApproveEntities(ImmutableSet.of(transferBillingEvent.createVKey())) .build()) @@ -418,8 +416,8 @@ public class DomainBaseTest extends EntityTestCase { GracePeriodStatus.ADD, domain.getRepoId(), fakeClock.nowUtc().plusDays(100), - "foo", - null)) + "TheRegistrar", + oneTimeBillKey)) .build(); DomainBase afterTransfer = domain.cloneProjectedAtTime(fakeClock.nowUtc().plusDays(1)); DateTime newExpirationTime = oldExpirationTime.plusYears(1); @@ -435,7 +433,7 @@ public class DomainBaseTest extends EntityTestCase { .nowUtc() .plusDays(1) .plus(Registry.get("com").getTransferGracePeriodLength()), - "winner", + "TheRegistrar", transferBillingEvent.createVKey(), afterTransfer.getGracePeriods().iterator().next().getGracePeriodId())); // If we project after the grace period expires all should be the same except the grace period. @@ -491,13 +489,13 @@ public class DomainBaseTest extends EntityTestCase { DomainBase beforeAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.minusDays(1)); assertThat(beforeAutoRenew.getLastEppUpdateTime()).isEqualTo(transferRequestDateTime); - assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("gaining"); + assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("TheRegistrar"); // If autorenew happens before transfer succeeds(before transfer grace period starts as well), // lastEppUpdateClientId should still be the current sponsor client id DomainBase afterAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.plusDays(1)); assertThat(afterAutoRenew.getLastEppUpdateTime()).isEqualTo(autorenewDateTime); - assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("losing"); + assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("NewRegistrar"); } @Test @@ -510,12 +508,12 @@ public class DomainBaseTest extends EntityTestCase { DomainBase beforeAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.minusDays(1)); assertThat(beforeAutoRenew.getLastEppUpdateTime()).isEqualTo(transferRequestDateTime); - assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("gaining"); + assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("TheRegistrar"); DomainBase afterTransferSuccess = domain.cloneProjectedAtTime(transferSuccessDateTime.plusDays(1)); assertThat(afterTransferSuccess.getLastEppUpdateTime()).isEqualTo(transferSuccessDateTime); - assertThat(afterTransferSuccess.getLastEppUpdateRegistrarId()).isEqualTo("gaining"); + assertThat(afterTransferSuccess.getLastEppUpdateRegistrarId()).isEqualTo("TheRegistrar"); } private void setupUnmodifiedDomain(DateTime oldExpirationTime) { @@ -542,7 +540,7 @@ public class DomainBaseTest extends EntityTestCase { DomainBase afterAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.plusDays(1)); assertThat(afterAutoRenew.getLastEppUpdateTime()).isEqualTo(autorenewDateTime); - assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("losing"); + assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("NewRegistrar"); } @Test @@ -911,20 +909,8 @@ public class DomainBaseTest extends EntityTestCase { @Test void testContactFields() { - VKey contact3Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id3") - .setRepoId("4-COM") - .build()) - .createVKey(); - VKey contact4Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id4") - .setRepoId("5-COM") - .build()) - .createVKey(); + VKey contact3Key = persistActiveContact("contact_id3").createVKey(); + VKey contact4Key = persistActiveContact("contact_id4").createVKey(); // Set all of the contacts. domain.setContactFields( diff --git a/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java b/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java index c7da11258..9b75c1a4f 100644 --- a/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java +++ b/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java @@ -26,12 +26,19 @@ import com.google.common.collect.Iterables; import com.googlecode.objectify.Key; import google.registry.model.EntityTestCase; import google.registry.model.contact.ContactResource; +import google.registry.testing.TmOverrideExtension; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link EppResourceIndex}. */ class EppResourceIndexTest extends EntityTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private ContactResource contact; @BeforeEach diff --git a/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java b/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java index fd292f02d..9c3026d98 100644 --- a/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java +++ b/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java @@ -27,13 +27,19 @@ import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.annotation.Cache; import google.registry.testing.AppEngineExtension; import google.registry.testing.InjectExtension; +import google.registry.testing.TmOverrideExtension; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link CommitLogBucket}. */ public class CommitLogBucketTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java b/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java index dfabb6c67..8766d3a6c 100644 --- a/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java +++ b/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java @@ -27,14 +27,19 @@ import com.googlecode.objectify.annotation.Parent; import google.registry.model.ImmutableObject; import google.registry.model.annotations.InCrossTld; import google.registry.model.common.EntityGroupRoot; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link DatastoreTransactionManager}. */ public class DatastoreTransactionManagerTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder() @@ -54,7 +59,6 @@ public class DatastoreTransactionManagerTest { } @Entity - @EntityForTesting @InCrossTld private static class InCrossTldTestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java b/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java index 472f373c8..062153b54 100644 --- a/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java +++ b/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java @@ -34,14 +34,20 @@ import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; import google.registry.testing.InjectExtension; import google.registry.testing.TestObject.TestVirtualObject; +import google.registry.testing.TmOverrideExtension; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests ensuring {@link Ofy} saves transactions to {@link CommitLogManifest}. */ public class OfyCommitLogTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder() diff --git a/core/src/test/java/google/registry/model/ofy/OfyTest.java b/core/src/test/java/google/registry/model/ofy/OfyTest.java index a2e2f6d3d..5d3456961 100644 --- a/core/src/test/java/google/registry/model/ofy/OfyTest.java +++ b/core/src/test/java/google/registry/model/ofy/OfyTest.java @@ -45,17 +45,17 @@ import google.registry.model.contact.ContactHistory; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DomainBase; import google.registry.model.eppcommon.Trid; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.model.reporting.HistoryEntry; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import google.registry.testing.AppEngineExtension; import google.registry.testing.DatabaseHelper; import google.registry.testing.FakeClock; +import google.registry.testing.TmOverrideExtension; import google.registry.util.SystemClock; import java.util.ConcurrentModificationException; import java.util.function.Supplier; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -64,6 +64,10 @@ public class OfyTest { private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ")); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().withClock(fakeClock).build(); @@ -178,7 +182,6 @@ public class OfyTest { /** Simple entity class with lifecycle callbacks. */ @com.googlecode.objectify.annotation.Entity - @EntityForTesting public static class LifecycleObject extends ImmutableObject { @Parent Key parent = getCrossTldKey(); @@ -437,12 +440,4 @@ public class OfyTest { // Test the normal loading again to verify that we've restored the original session unchanged. assertThat(auditedOfy().load().entity(someObject).now()).isEqualTo(someObject.asHistoryEntry()); } - - @Test - void testReadOnly_failsWrite() { - Ofy ofy = new Ofy(fakeClock); - DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(fakeClock); - assertThrows(ReadOnlyModeException.class, () -> ofy.save().entity(someObject).now()); - DatabaseHelper.removeDatabaseMigrationSchedule(); - } } diff --git a/core/src/test/java/google/registry/model/replay/EntityTest.java b/core/src/test/java/google/registry/model/replay/EntityTest.java deleted file mode 100644 index ebb4022a8..000000000 --- a/core/src/test/java/google/registry/model/replay/EntityTest.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; - -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import com.googlecode.objectify.annotation.Embed; -import com.googlecode.objectify.annotation.Parent; -import google.registry.model.ModelUtils; -import google.registry.model.common.GaeUserIdConverter; -import google.registry.persistence.VKey; -import google.registry.testing.DatastoreEntityExtension; -import io.github.classgraph.ClassGraph; -import io.github.classgraph.ClassInfo; -import io.github.classgraph.ClassInfoList; -import io.github.classgraph.ScanResult; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.lang.reflect.Method; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** - * Test to verify classes implement {@link SqlEntity} and {@link DatastoreEntity} when they should. - */ -public class EntityTest { - - @RegisterExtension - final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); - - private static final ImmutableSet> NON_CONVERTED_CLASSES = - ImmutableSet.of(GaeUserIdConverter.class); - - @Test - void testSqlEntityPersistence() { - try (ScanResult scanResult = scanForClasses()) { - // All javax.persistence entities must implement SqlEntity and vice versa - ImmutableSet javaxPersistenceClasses = - getAllClassesWithAnnotation(scanResult, javax.persistence.Entity.class.getName()); - ImmutableSet sqlEntityClasses = - getClassNames(scanResult.getClassesImplementing(SqlEntity.class.getName())); - assertThat(sqlEntityClasses).containsExactlyElementsIn(javaxPersistenceClasses); - - // All com.googlecode.objectify entities must implement DatastoreEntity and vice versa - ImmutableSet objectifyClasses = - getAllClassesWithAnnotation( - scanResult, com.googlecode.objectify.annotation.Entity.class.getName()); - ImmutableSet datastoreEntityClasses = - getClassNames(scanResult.getClassesImplementing(DatastoreEntity.class.getName())); - assertThat(datastoreEntityClasses).containsExactlyElementsIn(objectifyClasses); - } - } - - @Test - void testDatastoreEntityVKeyCreation() { - // For replication, we need to be able to convert from Key -> VKey for the relevant classes. - // This means that the relevant classes must have non-composite Objectify keys or must have a - // createVKey method - try (ScanResult scanResult = scanForClasses()) { - ImmutableSet> datastoreEntityClasses = - getClasses(scanResult.getClassesImplementing(DatastoreEntity.class.getName())); - // some classes aren't converted so they aren't relevant - ImmutableSet> vkeyConversionNecessaryClasses = - datastoreEntityClasses.stream() - .filter(clazz -> !DatastoreOnlyEntity.class.isAssignableFrom(clazz)) - .filter(clazz -> !NonReplicatedEntity.class.isAssignableFrom(clazz)) - .collect(toImmutableSet()); - - ImmutableSet.Builder> failedClasses = new ImmutableSet.Builder<>(); - for (Class clazz : vkeyConversionNecessaryClasses) { - if (hasKeyWithParent(clazz)) { - try { - Method createVKeyMethod = clazz.getMethod("createVKey", Key.class); - if (!createVKeyMethod.getReturnType().equals(VKey.class)) { - failedClasses.add(clazz); - } - } catch (NoSuchMethodException e) { - failedClasses.add(clazz); - } - } - } - assertWithMessage( - "Some DatastoreEntity classes with parents were missing createVKey methods: ") - .that(failedClasses.build()) - .isEmpty(); - } - } - - private boolean hasKeyWithParent(Class clazz) { - return ModelUtils.getAllFields(clazz).values().stream() - .anyMatch(field -> field.getAnnotation(Parent.class) != null); - } - - private ImmutableSet getAllClassesWithAnnotation( - ScanResult scanResult, String annotation) { - ImmutableSet.Builder result = new ImmutableSet.Builder<>(); - ClassInfoList classesWithAnnotation = scanResult.getClassesWithAnnotation(annotation); - result.addAll(getClassNames(classesWithAnnotation)); - classesWithAnnotation.stream() - .map(ClassInfo::getSubclasses) - .forEach(classInfoList -> result.addAll(getClassNames(classInfoList))); - return result.build(); - } - - private ImmutableSet> getClasses(ClassInfoList classInfoList) { - return classInfoList.stream() - .filter(ClassInfo::isStandardClass) - .map(ClassInfo::loadClass) - .filter( - clazz -> - !clazz.isAnnotationPresent(EntityForTesting.class) - && !clazz.isAnnotationPresent(Embed.class) - && !NON_CONVERTED_CLASSES.contains(clazz) - && !clazz.getName().contains("Test")) - .collect(toImmutableSet()); - } - - private ImmutableSet getClassNames(ClassInfoList classInfoList) { - return getClasses(classInfoList).stream().map(Class::getName).collect(toImmutableSet()); - } - - private ScanResult scanForClasses() { - return new ClassGraph() - .enableAnnotationInfo() - .ignoreClassVisibility() - .acceptPackages("google.registry") - .scan(); - } - - /** Entities that are solely used for testing, to avoid scanning them in {@link EntityTest}. */ - @Target(ElementType.TYPE) - @Retention(RetentionPolicy.RUNTIME) - public @interface EntityForTesting {} -} diff --git a/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java b/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java index 212d2b497..1dafe3bea 100644 --- a/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java +++ b/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java @@ -27,13 +27,14 @@ import com.googlecode.objectify.annotation.Entity; import google.registry.model.common.CrossTldSingleton; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.Ofy; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; import google.registry.testing.InjectExtension; +import google.registry.testing.TmOverrideExtension; import java.util.List; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -43,11 +44,14 @@ public class CommitLogRevisionsTranslatorFactoryTest { private static final DateTime START_TIME = DateTime.parse("2000-01-01TZ"); @Entity(name = "ClrtfTestEntity") - @EntityForTesting public static class TestObject extends CrossTldSingleton { ImmutableSortedMap> revisions = ImmutableSortedMap.of(); } + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder() diff --git a/core/src/test/java/google/registry/persistence/BillingVKeyTest.java b/core/src/test/java/google/registry/persistence/BillingVKeyTest.java index 2b68e5c23..b32b7a9f3 100644 --- a/core/src/test/java/google/registry/persistence/BillingVKeyTest.java +++ b/core/src/test/java/google/registry/persistence/BillingVKeyTest.java @@ -26,7 +26,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.billing.BillingEvent; import google.registry.model.common.EntityGroupRoot; import google.registry.model.domain.DomainBase; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.BillingVKey.BillingEventVKey; import google.registry.persistence.BillingVKey.BillingRecurrenceVKey; @@ -80,7 +79,6 @@ class BillingVKeyTest { assertThat(persisted).isEqualTo(original); } - @EntityForTesting @Entity @javax.persistence.Entity private static class BillingVKeyTestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java b/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java index b888eea5a..26f2be5ca 100644 --- a/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java +++ b/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java @@ -26,7 +26,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.common.EntityGroupRoot; import google.registry.model.domain.DomainBase; import google.registry.model.domain.DomainHistory.DomainHistoryId; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.AppEngineExtension; import google.registry.testing.DualDatabaseTest; @@ -82,7 +81,6 @@ class DomainHistoryVKeyTest { VKey.create(HistoryEntry.class, new DomainHistoryId("domainRepoId", 10L), ofyKey)); } - @EntityForTesting @Entity @javax.persistence.Entity(name = "TestEntity") private static class TestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java b/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java index 3dc744660..607f0fbe0 100644 --- a/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java +++ b/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java @@ -22,7 +22,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import com.google.common.collect.ImmutableSet; import google.registry.model.ImmutableObject; -import google.registry.model.replay.NonReplicatedEntity; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import java.lang.reflect.Method; @@ -169,7 +168,7 @@ class EntityCallbacksListenerTest { } @Entity(name = "TestEntity") - private static class TestEntity extends ParentEntity implements NonReplicatedEntity { + private static class TestEntity extends ParentEntity { @Id String name = "id"; int nonTransientField = 0; diff --git a/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java b/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java index 0c1428452..5767e2124 100644 --- a/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java @@ -22,7 +22,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import com.google.common.collect.ImmutableSet; import com.google.common.hash.BloomFilter; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import javax.persistence.Entity; @@ -49,7 +48,6 @@ class BloomFilterConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting public static class TestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java b/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java index 49ee2bfb0..9d01df530 100644 --- a/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java @@ -20,7 +20,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import static org.junit.jupiter.api.Assertions.assertThrows; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import javax.persistence.Entity; @@ -77,7 +76,6 @@ public class CurrencyUnitConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting public static class TestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java b/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java index dc4391057..36b6c27a5 100644 --- a/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java @@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.insertInDb; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import javax.persistence.Entity; @@ -88,7 +87,6 @@ public class DurationConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting public static class DurationTestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java b/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java index c06209f7f..4889c527d 100644 --- a/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java @@ -21,7 +21,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.VKey; import google.registry.testing.AppEngineExtension; import java.net.InetAddress; @@ -73,7 +72,6 @@ public class InetAddressSetConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting private static class InetAddressSetTestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java b/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java index b7869634c..ca0bb818a 100644 --- a/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java @@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableMap; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import java.math.BigDecimal; @@ -289,7 +288,6 @@ public class JodaMoneyConverterTest { // Override entity name to exclude outer-class name in table name. Not necessary if class is not // inner class. @Entity(name = "TestEntity") - @EntityForTesting public static class TestEntity extends ImmutableObject { @Id String name = "id"; @@ -307,7 +305,6 @@ public class JodaMoneyConverterTest { // See comments on the annotation for TestEntity above for reason. @Entity(name = "ComplexTestEntity") - @EntityForTesting // This entity is used to test column override for embedded fields and collections. public static class ComplexTestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java b/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java index 733a538e3..15a1e51ac 100644 --- a/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java @@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.insertInDb; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest; import google.registry.persistence.VKey; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; @@ -63,7 +62,6 @@ public class LocalDateConverterTest { /** Override entity name to avoid the nested class reference. */ @Entity(name = "LocalDateConverterTestEntity") - @EntityTest.EntityForTesting private static class LocalDateConverterTestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java index 16f2a47ac..53dfbc2c2 100644 --- a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java +++ b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import com.google.common.io.Resources; -import google.registry.model.common.DatabaseMigrationStateSchedule; import google.registry.persistence.HibernateSchemaExporter; import google.registry.persistence.NomulusPostgreSql; import google.registry.persistence.PersistenceModule; @@ -168,7 +167,7 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft if (!includeNomulusSchema) { File tempSqlFile = File.createTempFile("tempSqlFile", ".sql"); tempSqlFile.deleteOnExit(); - exporter.export(getTestEntities(), tempSqlFile); + exporter.export(extraEntityClasses, tempSqlFile); executeSql(new String(Files.readAllBytes(tempSqlFile.toPath()), StandardCharsets.UTF_8)); } assertReasonableNumDbConnections(); @@ -238,14 +237,16 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft ResultSet rs = statement.executeQuery( "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';"); - ImmutableList.Builder tableNames = new ImmutableList.Builder<>(); + ImmutableList.Builder tableNamesBuilder = new ImmutableList.Builder<>(); while (rs.next()) { - tableNames.add('"' + rs.getString(1) + '"'); + tableNamesBuilder.add('"' + rs.getString(1) + '"'); + } + ImmutableList tableNames = tableNamesBuilder.build(); + if (!tableNames.isEmpty()) { + String sql = + String.format("TRUNCATE %s RESTART IDENTITY CASCADE", Joiner.on(',').join(tableNames)); + executeSql(sql); } - String sql = - String.format( - "TRUNCATE %s RESTART IDENTITY CASCADE", Joiner.on(',').join(tableNames.build())); - executeSql(sql); } catch (Exception e) { throw new RuntimeException(e); } @@ -344,17 +345,7 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft descriptor.getManagedClassNames().addAll(nonEntityClasses); } - getTestEntities().stream().map(Class::getName).forEach(descriptor::addClasses); + extraEntityClasses.stream().map(Class::getName).forEach(descriptor::addClasses); return Bootstrap.getEntityManagerFactoryBuilder(descriptor, properties).build(); } - - private ImmutableList> getTestEntities() { - // We have to add the DatabaseMigrationStateSchedule and TransactionEntity classes to extra - // entities, as they are required by the transaction manager factory and transaction replication - // mechanism, respectively. - return Stream.concat( - extraEntityClasses.stream(), - Stream.of(DatabaseMigrationStateSchedule.class, TransactionEntity.class)) - .collect(toImmutableList()); - } } diff --git a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java index dd04389a1..4dc11a2d1 100644 --- a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java +++ b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java @@ -36,7 +36,6 @@ import google.registry.persistence.VKey; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import google.registry.testing.DatabaseHelper; import google.registry.testing.FakeClock; -import google.registry.testing.TmOverrideExtension; import java.io.Serializable; import java.math.BigInteger; import java.sql.SQLException; @@ -49,7 +48,6 @@ import javax.persistence.IdClass; import javax.persistence.OptimisticLockException; import javax.persistence.RollbackException; import org.hibernate.exception.JDBCConnectionException; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -84,10 +82,6 @@ class JpaTransactionManagerImplTest { TestEntity.class, TestCompoundIdEntity.class, TestNamedCompoundIdEntity.class) .buildUnitTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - @Test void transact_succeeds() { assertPersonEmpty(); diff --git a/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java b/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java index cfbea25ae..2506c922d 100644 --- a/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java +++ b/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java @@ -340,16 +340,6 @@ public class ReplicaSimulatingJpaTransactionManager implements JpaTransactionMan return delegate.isOfy(); } - @Override - public void putIgnoringReadOnlyWithoutBackup(Object entity) { - delegate.putIgnoringReadOnlyWithoutBackup(entity); - } - - @Override - public void deleteIgnoringReadOnlyWithoutBackup(VKey key) { - delegate.deleteIgnoringReadOnlyWithoutBackup(key); - } - @Override public void assertDelete(VKey key) { delegate.assertDelete(key); diff --git a/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java b/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java index f79c4fc91..a654e0852 100644 --- a/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java +++ b/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java @@ -30,11 +30,8 @@ import com.googlecode.objectify.annotation.Id; import google.registry.model.ImmutableObject; import google.registry.model.ofy.DatastoreTransactionManager; import google.registry.model.ofy.Ofy; -import google.registry.model.replay.NonReplicatedEntity; import google.registry.persistence.VKey; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; import google.registry.testing.DualDatabaseTest; import google.registry.testing.FakeClock; import google.registry.testing.InjectExtension; @@ -409,13 +406,6 @@ public class TransactionManagerTest { assertThat(tm().transact(() -> tm().loadByKey(theEntity.key())).data).isEqualTo("foo"); } - @TestOfyAndSql - void testReadOnly_writeFails() { - DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(fakeClock); - assertThrows(ReadOnlyModeException.class, () -> tm().transact(() -> tm().put(theEntity))); - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - private static void assertEntityExists(TestEntity entity) { assertThat(tm().transact(() -> tm().exists(entity))).isTrue(); } @@ -449,7 +439,7 @@ public class TransactionManagerTest { @Entity(name = "TxnMgrTestEntity") @javax.persistence.Entity(name = "TestEntity") - private static class TestEntity extends TestEntityBase implements NonReplicatedEntity { + private static class TestEntity extends TestEntityBase { private String data; diff --git a/core/src/test/java/google/registry/persistence/transaction/TransactionTest.java b/core/src/test/java/google/registry/persistence/transaction/TransactionTest.java deleted file mode 100644 index b6e200442..000000000 --- a/core/src/test/java/google/registry/persistence/transaction/TransactionTest.java +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.googlecode.objectify.Key; -import com.googlecode.objectify.annotation.Entity; -import com.googlecode.objectify.annotation.Id; -import google.registry.model.ImmutableObject; -import google.registry.model.ofy.CommitLogManifest; -import google.registry.model.ofy.CommitLogMutation; -import google.registry.model.ofy.Ofy; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.StreamCorruptedException; -import java.util.Comparator; -import java.util.List; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -class TransactionTest { - - private final FakeClock fakeClock = - new FakeClock(DateTime.parse("2000-01-01TZ")).setAutoIncrementByOneMilli(); - - @RegisterExtension - final AppEngineExtension appEngine = - AppEngineExtension.builder() - .withDatastoreAndCloudSql() - .withClock(fakeClock) - .withOfyTestEntities(TestEntity.class) - .withJpaUnitTestEntities(TestEntity.class) - .build(); - - @RegisterExtension public final InjectExtension inject = new InjectExtension(); - - private TestEntity fooEntity, barEntity; - - @BeforeEach - void beforeEach() { - inject.setStaticField(Ofy.class, "clock", fakeClock); - fooEntity = new TestEntity("foo"); - barEntity = new TestEntity("bar"); - } - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @Test - void testTransactionReplay() { - Transaction txn = new Transaction.Builder().addUpdate(fooEntity).addUpdate(barEntity).build(); - txn.writeToDatastore(); - - ofyTm() - .transact( - () -> { - assertThat(ofyTm().loadByKey(fooEntity.key())).isEqualTo(fooEntity); - assertThat(ofyTm().loadByKey(barEntity.key())).isEqualTo(barEntity); - }); - - txn = new Transaction.Builder().addDelete(barEntity.key()).build(); - txn.writeToDatastore(); - assertThat(ofyTm().exists(barEntity.key())).isEqualTo(false); - - assertThat( - auditedOfy().load().type(CommitLogMutation.class).list().stream() - .map(clm -> auditedOfy().load().fromEntity(clm.getEntity())) - .collect(toImmutableSet())) - .containsExactly(fooEntity, barEntity); - List manifests = auditedOfy().load().type(CommitLogManifest.class).list(); - manifests.sort(Comparator.comparing(CommitLogManifest::getCommitTime)); - assertThat(manifests.get(0).getDeletions()).isEmpty(); - assertThat(manifests.get(1).getDeletions()).containsExactly(Key.create(barEntity)); - } - - @Test - void testSerialization() throws Exception { - Transaction txn = new Transaction.Builder().addUpdate(barEntity).build(); - txn.writeToDatastore(); - - txn = new Transaction.Builder().addUpdate(fooEntity).addDelete(barEntity.key()).build(); - txn = Transaction.deserialize(txn.serialize()); - - txn.writeToDatastore(); - - ofyTm() - .transact( - () -> { - assertThat(ofyTm().loadByKey(fooEntity.key())).isEqualTo(fooEntity); - assertThat(ofyTm().exists(barEntity.key())).isEqualTo(false); - }); - } - - @Test - void testDeserializationErrors() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(baos); - out.writeInt(12345); - out.close(); - assertThrows(IllegalArgumentException.class, () -> Transaction.deserialize(baos.toByteArray())); - - // Test with a short byte array. - assertThrows( - StreamCorruptedException.class, () -> Transaction.deserialize(new byte[] {1, 2, 3, 4})); - } - - @Test - void testTransactionSerialization() throws IOException { - DatabaseHelper.setMigrationScheduleToSqlPrimary(fakeClock); - jpaTm() - .transact( - () -> { - jpaTm().insert(fooEntity); - jpaTm().insert(barEntity); - }); - TransactionEntity txnEnt = - jpaTm().transact(() -> jpaTm().loadByKey(VKey.createSql(TransactionEntity.class, 1L))); - Transaction txn = Transaction.deserialize(txnEnt.getContents()); - txn.writeToDatastore(); - ofyTm() - .transact( - () -> { - assertThat(ofyTm().loadByKey(fooEntity.key())).isEqualTo(fooEntity); - assertThat(ofyTm().loadByKey(barEntity.key())).isEqualTo(barEntity); - }); - - // Verify that no transaction was persisted for the load transaction. - assertThat( - jpaTm().transact(() -> jpaTm().exists(VKey.createSql(TransactionEntity.class, 2L)))) - .isFalse(); - } - - @Test - void testTransactionSerializationDisabledByDefault() { - jpaTm() - .transact( - () -> { - jpaTm().insert(fooEntity); - jpaTm().insert(barEntity); - }); - assertThat(jpaTm().transact(() -> jpaTm().exists(VKey.createSql(TransactionEntity.class, 1L)))) - .isFalse(); - } - - @Entity(name = "TxnTestEntity") - @javax.persistence.Entity(name = "TestEntity") - private static class TestEntity extends ImmutableObject { - @Id @javax.persistence.Id private String name; - - private TestEntity() {} - - private TestEntity(String name) { - this.name = name; - } - - public VKey key() { - return VKey.create(TestEntity.class, name, Key.create(this)); - } - } -} diff --git a/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java b/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java index 011315c10..f99fdc87e 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java @@ -56,6 +56,7 @@ import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeLockHandler; import google.registry.testing.FakeResponse; import google.registry.testing.InjectExtension; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import google.registry.tldconfig.idn.IdnTableEnum; import google.registry.xjc.XjcXmlTransformer; @@ -84,12 +85,17 @@ import org.joda.time.DateTimeConstants; import org.joda.time.Duration; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link RdeStagingAction} in Datastore. */ public class RdeStagingActionDatastoreTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final BlobId XML_FILE = BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde"); private static final BlobId LENGTH_FILE = diff --git a/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java b/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java index 170e4c87c..685635b28 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java @@ -27,10 +27,12 @@ import com.google.common.collect.ImmutableSetMultimap; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.Registrar.State; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; import google.registry.xml.ValidationMode; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -53,6 +55,10 @@ class RdeStagingMapperTest { private ArgumentCaptor depositFragmentCaptor = ArgumentCaptor.forClass(DepositFragment.class); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension AppEngineExtension appEngineExtension = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java index 2361bba6f..dcc9cacd8 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java @@ -43,6 +43,7 @@ import google.registry.testing.CloudTasksHelper; import google.registry.testing.CloudTasksHelper.TaskMatcher; import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeLockHandler; +import google.registry.testing.TmOverrideExtension; import google.registry.xml.ValidationMode; import java.io.IOException; import java.util.Iterator; @@ -52,12 +53,17 @@ import org.bouncycastle.openpgp.PGPPublicKey; import org.joda.time.DateTime; import org.joda.time.Duration; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link RdeStagingReducer}. */ class RdeStagingReducerTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension AppEngineExtension appEngineExtension = AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); diff --git a/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java b/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java index c82719111..a205c73de 100644 --- a/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java +++ b/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java @@ -21,13 +21,11 @@ import static google.registry.xjc.XjcXmlTransformer.marshalStrict; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableList; -import google.registry.model.ofy.Ofy; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.Registrar.State; import google.registry.model.registrar.RegistrarAddress; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; import google.registry.xjc.rderegistrar.XjcRdeRegistrar; import google.registry.xjc.rderegistrar.XjcRdeRegistrarAddrType; import google.registry.xjc.rderegistrar.XjcRdeRegistrarPostalInfoEnumType; @@ -47,11 +45,11 @@ import org.junit.jupiter.api.extension.RegisterExtension; */ public class RegistrarToXjcConverterTest { + private final FakeClock clock = new FakeClock(DateTime.parse("2013-01-01T00:00:00Z")); + @RegisterExtension public final AppEngineExtension appEngine = - AppEngineExtension.builder().withDatastoreAndCloudSql().build(); - - @RegisterExtension public final InjectExtension inject = new InjectExtension(); + AppEngineExtension.builder().withDatastoreAndCloudSql().withClock(clock).build(); private Registrar registrar; @@ -86,9 +84,8 @@ public class RegistrarToXjcConverterTest { .setWhoisServer("whois.goblinmen.example") .setUrl("http://www.goblinmen.example") .build(); - FakeClock clock = new FakeClock(DateTime.parse("2013-01-01T00:00:00Z")); - inject.setStaticField(Ofy.class, "clock", clock); registrar = cloneAndSetAutoTimestamps(registrar); // Set the creation time in 2013. + registrar = registrar.asBuilder().setLastUpdateTime(null).build(); clock.setTo(DateTime.parse("2014-01-01T00:00:00Z")); registrar = cloneAndSetAutoTimestamps(registrar); // Set the update time in 2014. } diff --git a/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java b/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java index 3e6f47875..8107cd1c4 100644 --- a/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java +++ b/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.when; import com.google.cloud.tasks.v2.HttpMethod; import com.google.common.net.MediaType; import google.registry.beam.BeamActionTestBase; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.reporting.ReportingModule; import google.registry.testing.AppEngineExtension; import google.registry.testing.CloudTasksHelper; @@ -62,7 +61,6 @@ class GenerateInvoicesActionTest extends BeamActionTestBase { "billing_bucket", "REG-INV", true, - PrimaryDatabase.DATASTORE, new YearMonth(2017, 10), emailUtils, cloudTasksUtils, @@ -97,7 +95,6 @@ class GenerateInvoicesActionTest extends BeamActionTestBase { "billing_bucket", "REG-INV", false, - PrimaryDatabase.DATASTORE, new YearMonth(2017, 10), emailUtils, cloudTasksUtils, @@ -122,7 +119,6 @@ class GenerateInvoicesActionTest extends BeamActionTestBase { "billing_bucket", "REG-INV", false, - PrimaryDatabase.DATASTORE, new YearMonth(2017, 10), emailUtils, cloudTasksUtils, diff --git a/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java b/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java index 48eddef0b..de59bec9e 100644 --- a/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java +++ b/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.when; import com.google.cloud.tasks.v2.HttpMethod; import com.google.common.net.MediaType; import google.registry.beam.BeamActionTestBase; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.reporting.ReportingModule; import google.registry.testing.AppEngineExtension; import google.registry.testing.CloudTasksHelper; @@ -57,7 +56,6 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase { "gs://reporting-project/reporting-bucket/", "api_key/a", clock.nowUtc().toLocalDate(), - PrimaryDatabase.DATASTORE, true, clock, response, @@ -81,7 +79,6 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase { "gs://reporting-project/reporting-bucket/", "api_key/a", clock.nowUtc().toLocalDate(), - PrimaryDatabase.DATASTORE, true, clock, response, @@ -115,7 +112,6 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase { "gs://reporting-project/reporting-bucket/", "api_key/a", clock.nowUtc().toLocalDate(), - PrimaryDatabase.DATASTORE, false, clock, response, diff --git a/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java b/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java index b90996d4f..bf7a15752 100644 --- a/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java +++ b/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java @@ -27,7 +27,6 @@ import google.registry.model.registrar.RegistrarContact; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension; import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.TmOverrideExtension; import google.registry.util.SerializeUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; @@ -45,10 +44,6 @@ class RegistrarContactTest { JpaIntegrationWithCoverageExtension jpa = new JpaTestExtensions.Builder().buildIntegrationWithCoverageExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private Registrar testRegistrar; private RegistrarContact testRegistrarPoc; diff --git a/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java b/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java index 8a559359d..6ab068d7a 100644 --- a/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java +++ b/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java @@ -30,7 +30,6 @@ import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension; import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; -import google.registry.testing.TmOverrideExtension; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; @@ -50,10 +49,6 @@ public class RegistrarDaoTest { JpaIntegrationWithCoverageExtension jpa = new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private final VKey registrarKey = VKey.createSql(Registrar.class, "registrarId"); private Registrar testRegistrar; diff --git a/core/src/test/java/google/registry/schema/replay/SqlEntityTest.java b/core/src/test/java/google/registry/schema/replay/SqlEntityTest.java deleted file mode 100644 index 09502c332..000000000 --- a/core/src/test/java/google/registry/schema/replay/SqlEntityTest.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.schema.replay; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; - -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import google.registry.model.registrar.RegistrarContact.RegistrarPocId; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.TmOverrideExtension; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Unit tests for {@link SqlEntity#getPrimaryKeyString}. */ -public class SqlEntityTest { - - @RegisterExtension - @Order(1) - final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); - - @RegisterExtension - final AppEngineExtension database = - new AppEngineExtension.Builder().withCloudSql().withoutCannedData().build(); - - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - - @BeforeEach - void setup() throws Exception { - AppEngineExtension.loadInitialData(); - } - - @Test - void getPrimaryKeyString_oneIdColumn() { - // AppEngineExtension canned data: Registrar1 - assertThat( - tm().transact(() -> tm().loadByKey(Registrar.createVKey("NewRegistrar"))) - .getPrimaryKeyString()) - .contains("NewRegistrar"); - } - - @Test - void getPrimaryKeyString_multiId() { - // AppEngineExtension canned data: RegistrarContact1 - VKey key = - VKey.createSql( - RegistrarContact.class, new RegistrarPocId("janedoe@theregistrar.com", "NewRegistrar")); - String expected = "emailAddress=janedoe@theregistrar.com\n registrarId=NewRegistrar"; - assertThat(tm().transact(() -> tm().loadByKey(key)).getPrimaryKeyString()).contains(expected); - } -} diff --git a/core/src/test/java/google/registry/testing/DatabaseHelper.java b/core/src/test/java/google/registry/testing/DatabaseHelper.java index 88a0dec37..c4fa20414 100644 --- a/core/src/test/java/google/registry/testing/DatabaseHelper.java +++ b/core/src/test/java/google/registry/testing/DatabaseHelper.java @@ -71,8 +71,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.billing.BillingEvent; import google.registry.model.billing.BillingEvent.Flag; import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; import google.registry.model.contact.ContactAuthInfo; import google.registry.model.contact.ContactHistory; import google.registry.model.contact.ContactResource; @@ -126,7 +124,6 @@ import org.joda.money.CurrencyUnit; import org.joda.money.Money; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.joda.time.Duration; /** Static utils for setting up test resources. */ public class DatabaseHelper { @@ -1210,7 +1207,7 @@ public class DatabaseHelper { * entities. */ public static void insertSimpleResources(final Iterable resources) { - tm().transact(() -> tm().insertAllWithoutBackup(ImmutableList.copyOf(resources))); + tm().transact(() -> tm().putAllWithoutBackup(ImmutableList.copyOf(resources))); maybeAdvanceClock(); // Force the session to be cleared so that when we read it back, we read from Datastore // and not from the transaction's session cache. @@ -1226,18 +1223,10 @@ public class DatabaseHelper { /** Force the create and update timestamps to get written into the resource. */ public static R cloneAndSetAutoTimestamps(final R resource) { - R result; - if (tm().isOfy()) { - result = - tm().transact( - () -> auditedOfy().load().fromEntity(auditedOfy().save().toEntity(resource))); - } else { - // We have to separate the read and write operation into different transactions - // otherwise JPA would just return the input entity instead of actually creating a - // clone. - tm().transact(() -> tm().put(resource)); - result = tm().transact(() -> tm().loadByEntity(resource)); - } + // We have to separate the read and write operation into different transactions otherwise JPA + // would just return the input entity instead of actually creating a clone. + tm().transact(() -> tm().put(resource)); + R result = tm().transact(() -> tm().loadByEntity(resource)); maybeAdvanceClock(); return result; } @@ -1408,104 +1397,5 @@ public class DatabaseHelper { return entity; } - /** - * Sets a DATASTORE_PRIMARY_NO_ASYNC state on the {@link DatabaseMigrationStateSchedule}. - * - *

In order to allow for tests to manipulate the clock how they need, we start the transitions - * one millisecond after the clock's current time (in case the clock's current value is - * START_OF_TIME). We then advance the clock one second so that we're in the - * DATASTORE_PRIMARY_READ_ONLY phase. - * - *

We must use the current time, otherwise the setting of the migration state will fail due to - * an invalid transition. - */ - public static void setMigrationScheduleToDatastorePrimaryNoAsync(FakeClock fakeClock) { - DateTime now = fakeClock.nowUtc(); - jpaTm() - .transact( - () -> - DatabaseMigrationStateSchedule.set( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMillis(1), - MigrationState.DATASTORE_PRIMARY, - now.plusMillis(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC))); - fakeClock.advanceBy(Duration.standardSeconds(1)); - } - - /** - * Sets a DATASTORE_PRIMARY_READ_ONLY state on the {@link DatabaseMigrationStateSchedule}. - * - *

In order to allow for tests to manipulate the clock how they need, we start the transitions - * one millisecond after the clock's current time (in case the clock's current value is - * START_OF_TIME). We then advance the clock one second so that we're in the - * DATASTORE_PRIMARY_READ_ONLY phase. - * - *

We must use the current time, otherwise the setting of the migration state will fail due to - * an invalid transition. - */ - public static void setMigrationScheduleToDatastorePrimaryReadOnly(FakeClock fakeClock) { - DateTime now = fakeClock.nowUtc(); - jpaTm() - .transact( - () -> - DatabaseMigrationStateSchedule.set( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMillis(1), - MigrationState.DATASTORE_PRIMARY, - now.plusMillis(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusMillis(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY))); - fakeClock.advanceBy(Duration.standardSeconds(1)); - } - - /** - * Sets a SQL_PRIMARY state on the {@link DatabaseMigrationStateSchedule}. - * - *

In order to allow for tests to manipulate the clock how they need, we start the transitions - * one millisecond after the clock's current time (in case the clock's current value is - * START_OF_TIME). We then advance the clock one second so that we're in the SQL_PRIMARY phase. - * - *

We must use the current time, otherwise the setting of the migration state will fail due to - * an invalid transition. - */ - public static void setMigrationScheduleToSqlPrimary(FakeClock fakeClock) { - DateTime now = fakeClock.nowUtc(); - jpaTm() - .transact( - () -> - DatabaseMigrationStateSchedule.set( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMillis(1), - MigrationState.DATASTORE_PRIMARY, - now.plusMillis(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusMillis(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - now.plusMillis(4), - MigrationState.SQL_PRIMARY))); - fakeClock.advanceBy(Duration.standardSeconds(1)); - } - - /** Removes the database migration schedule, in essence transitioning to DATASTORE_ONLY. */ - public static void removeDatabaseMigrationSchedule() { - // use the raw calls because going SQL_PRIMARY -> DATASTORE_ONLY is not valid - jpaTm() - .transact( - () -> - jpaTm() - .putIgnoringReadOnlyWithoutBackup( - new DatabaseMigrationStateSchedule( - DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP))); - DatabaseMigrationStateSchedule.CACHE.invalidateAll(); - } - private DatabaseHelper() {} } diff --git a/core/src/test/java/google/registry/testing/TestObject.java b/core/src/test/java/google/registry/testing/TestObject.java index d54984164..7491a8a26 100644 --- a/core/src/test/java/google/registry/testing/TestObject.java +++ b/core/src/test/java/google/registry/testing/TestObject.java @@ -23,16 +23,13 @@ import com.googlecode.objectify.annotation.Parent; import google.registry.model.ImmutableObject; import google.registry.model.annotations.VirtualEntity; import google.registry.model.common.EntityGroupRoot; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.VKey; import javax.persistence.Transient; /** A test model object that can be persisted in any entity group. */ @Entity @javax.persistence.Entity -@EntityForTesting -public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity { +public class TestObject extends ImmutableObject { @Parent @Transient Key parent; @@ -75,7 +72,6 @@ public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity /** A test @VirtualEntity model object, which should not be persisted. */ @Entity @VirtualEntity - @EntityForTesting public static class TestVirtualObject extends ImmutableObject { @Id String id; diff --git a/core/src/test/java/google/registry/testing/TmOverrideExtension.java b/core/src/test/java/google/registry/testing/TmOverrideExtension.java index 247285423..6c6d658df 100644 --- a/core/src/test/java/google/registry/testing/TmOverrideExtension.java +++ b/core/src/test/java/google/registry/testing/TmOverrideExtension.java @@ -14,9 +14,9 @@ package google.registry.testing; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.persistence.transaction.TransactionManager; import google.registry.persistence.transaction.TransactionManagerFactory; import org.junit.jupiter.api.extension.AfterEachCallback; @@ -35,35 +35,17 @@ import org.junit.jupiter.api.extension.ExtensionContext; *

This extension is incompatible with {@link DualDatabaseTest}. Use either that or this, but not * both. */ +@DeleteAfterMigration public final class TmOverrideExtension implements BeforeEachCallback, AfterEachCallback { - private static enum TmOverride { - OFY, - JPA; - } - - private final TmOverride tmOverride; - - private TmOverrideExtension(TmOverride tmOverride) { - this.tmOverride = tmOverride; - } - /** Use the {@link google.registry.model.ofy.DatastoreTransactionManager} for all tests. */ public static TmOverrideExtension withOfy() { - return new TmOverrideExtension(TmOverride.OFY); - } - - /** - * Use the {@link google.registry.persistence.transaction.JpaTransactionManager} for all tests. - */ - public static TmOverrideExtension withJpa() { - return new TmOverrideExtension(TmOverride.JPA); + return new TmOverrideExtension(); } @Override public void beforeEach(ExtensionContext context) { - TransactionManagerFactory.setTmOverrideForTest( - tmOverride == TmOverride.OFY ? ofyTm() : jpaTm()); + TransactionManagerFactory.setTmOverrideForTest(ofyTm()); } @Override diff --git a/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java b/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java index 6a137da2f..4d3148e48 100644 --- a/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java +++ b/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java @@ -89,13 +89,18 @@ class NordnUploadActionTest { private static final String LOCATION_URL = "http://trololol"; + private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z")); + @RegisterExtension public final AppEngineExtension appEngine = - AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); + AppEngineExtension.builder() + .withDatastoreAndCloudSql() + .withClock(clock) + .withTaskQueue() + .build(); @RegisterExtension public final InjectExtension inject = new InjectExtension(); - - private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z")); + private final LordnRequestInitializer lordnRequestInitializer = new LordnRequestInitializer(Optional.of("attack")); private final NordnUploadAction action = new NordnUploadAction(); diff --git a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java index dbbc7f043..5a8f0840e 100644 --- a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java +++ b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.io.Resources; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.testing.DatastoreEntityExtension; import google.registry.tools.EntityWrapper.Property; import java.io.ByteArrayOutputStream; @@ -32,6 +33,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; +@DeleteAfterMigration public class CompareDbBackupsTest { private static final int BASE_ID = 1001; diff --git a/core/src/test/java/google/registry/tools/CreateTldCommandTest.java b/core/src/test/java/google/registry/tools/CreateTldCommandTest.java index 60deb80f4..bcbc05fdf 100644 --- a/core/src/test/java/google/registry/tools/CreateTldCommandTest.java +++ b/core/src/test/java/google/registry/tools/CreateTldCommandTest.java @@ -55,9 +55,9 @@ class CreateTldCommandTest extends CommandTestCase { @Test void testSuccess() throws Exception { - DateTime before = DateTime.now(UTC); + DateTime before = fakeClock.nowUtc(); runCommandForced("xn--q9jyb4c", "--roid_suffix=Q9JYB4C", "--dns_writers=FooDnsWriter"); - DateTime after = DateTime.now(UTC); + DateTime after = fakeClock.nowUtc(); Registry registry = Registry.get("xn--q9jyb4c"); assertThat(registry).isNotNull(); diff --git a/core/src/test/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommandTest.java b/core/src/test/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommandTest.java deleted file mode 100644 index 40c433c2d..000000000 --- a/core/src/test/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommandTest.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.loadAllOf; -import static google.registry.testing.DatabaseHelper.persistActiveDomain; -import static google.registry.testing.DatabaseHelper.persistResource; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.joda.money.CurrencyUnit.USD; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.DomainHistory; -import google.registry.model.domain.Period; -import google.registry.model.eppcommon.Trid; -import google.registry.model.poll.PollMessage; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.transfer.DomainTransferData; -import org.joda.money.Money; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * Unit tests for {@link DedupeOneTimeBillingEventIdsCommand}. - * - *

Note that these are _not_ dual database tests even though the action has been converted. The - * dedupe was strictly a one-time event that needed to be done prior to moving to SQL. It should no - * longer be necessary and we may want to simply remove the command. - */ -class DedupeOneTimeBillingEventIdsCommandTest - extends CommandTestCase { - - DomainBase domain; - DomainHistory historyEntry; - PollMessage.Autorenew autorenewToResave; - BillingEvent.OneTime billingEventToResave; - - @BeforeEach - void beforeEach() { - createTld("foobar"); - domain = persistActiveDomain("foo.foobar"); - historyEntry = persistHistoryEntry(domain); - autorenewToResave = persistAutorenewPollMessage(historyEntry); - billingEventToResave = persistBillingEvent(historyEntry); - } - - @Test - void resaveBillingEvent_succeeds() throws Exception { - runCommand( - "--force", - "--key_paths_file", - writeToNamedTmpFile("keypath.txt", getKeyPathLiteral(billingEventToResave))); - - int count = 0; - for (BillingEvent.OneTime billingEvent : loadAllOf(BillingEvent.OneTime.class)) { - count++; - assertThat(billingEvent.getId()).isNotEqualTo(billingEventToResave.getId()); - assertThat(billingEvent.asBuilder().setId(billingEventToResave.getId()).build()) - .isEqualTo(billingEventToResave); - } - assertThat(count).isEqualTo(1); - } - - @Test - void resaveBillingEvent_failsWhenReferredByDomain() { - persistResource( - domain - .asBuilder() - .setTransferData( - new DomainTransferData.Builder() - .setServerApproveEntities(ImmutableSet.of(billingEventToResave.createVKey())) - .build()) - .build()); - - assertThrows( - IllegalStateException.class, - () -> - runCommand( - "--force", - "--key_paths_file", - writeToNamedTmpFile("keypath.txt", getKeyPathLiteral(billingEventToResave)))); - } - - private PollMessage.Autorenew persistAutorenewPollMessage(HistoryEntry historyEntry) { - return persistResource( - new PollMessage.Autorenew.Builder() - .setRegistrarId("TheRegistrar") - .setEventTime(fakeClock.nowUtc()) - .setMsg("Test poll message") - .setParent(historyEntry) - .setAutorenewEndTime(fakeClock.nowUtc().plusDays(365)) - .setTargetId("foobar.foo") - .build()); - } - - private BillingEvent.OneTime persistBillingEvent(DomainHistory historyEntry) { - return persistResource( - new BillingEvent.OneTime.Builder() - .setRegistrarId("a registrar") - .setTargetId("foo.tld") - .setParent(historyEntry) - .setReason(Reason.CREATE) - .setFlags(ImmutableSet.of(BillingEvent.Flag.ANCHOR_TENANT)) - .setPeriodYears(2) - .setCost(Money.of(USD, 1)) - .setEventTime(fakeClock.nowUtc()) - .setBillingTime(fakeClock.nowUtc().plusDays(5)) - .build()); - } - - private DomainHistory persistHistoryEntry(DomainBase parent) { - return persistResource( - new DomainHistory.Builder() - .setDomain(parent) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .setPeriod(Period.create(1, Period.Unit.YEARS)) - .setXmlBytes("".getBytes(UTF_8)) - .setModificationTime(fakeClock.nowUtc()) - .setRegistrarId("foo") - .setTrid(Trid.create("ABC-123", "server-trid")) - .setBySuperuser(false) - .setReason("reason") - .setRequestedByRegistrar(false) - .build()); - } - - private static String getKeyPathLiteral(Object entity) { - Key key = Key.create(entity); - return String.format( - "\"DomainBase\", \"%s\", \"HistoryEntry\", %s, \"%s\", %s", - key.getParent().getParent().getName(), key.getParent().getId(), key.getKind(), key.getId()); - } -} diff --git a/core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java b/core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java deleted file mode 100644 index 7b3a910bc..000000000 --- a/core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.util.DateTimeUtils.START_OF_TIME; - -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.DualDatabaseTest; -import google.registry.testing.TestOfyAndSql; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; - -/** Tests for {@link GetDatabaseMigrationStateCommand}. */ -@DualDatabaseTest -public class GetDatabaseMigrationStateCommandTest - extends CommandTestCase { - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @TestOfyAndSql - void testInitial_returnsDatastoreOnly() throws Exception { - runCommand(); - assertStdoutIs( - String.format("Current migration schedule: %s\n", DEFAULT_TRANSITION_MAP.toValueMap())); - } - - @TestOfyAndSql - void testFullSchedule() throws Exception { - DateTime now = fakeClock.nowUtc(); - ImmutableSortedMap transitions = - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusHours(1), - MigrationState.DATASTORE_PRIMARY, - now.plusHours(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusHours(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - now.plusHours(4), - MigrationState.SQL_PRIMARY, - now.plusHours(5), - MigrationState.SQL_ONLY); - jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions)); - runCommand(); - assertStdoutIs(String.format("Current migration schedule: %s\n", transitions)); - } -} diff --git a/core/src/test/java/google/registry/tools/GetDomainCommandTest.java b/core/src/test/java/google/registry/tools/GetDomainCommandTest.java index 3c5c15b24..c64e41575 100644 --- a/core/src/test/java/google/registry/tools/GetDomainCommandTest.java +++ b/core/src/test/java/google/registry/tools/GetDomainCommandTest.java @@ -19,19 +19,15 @@ import static google.registry.testing.DatabaseHelper.newDomainBase; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistDeletedDomain; import static google.registry.testing.DatabaseHelper.persistResource; -import static org.joda.time.DateTimeZone.UTC; import static org.junit.jupiter.api.Assertions.assertThrows; import com.beust.jcommander.ParameterException; -import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** Unit tests for {@link GetDomainCommand}. */ class GetDomainCommandTest extends CommandTestCase { - private DateTime now = DateTime.now(UTC); - @BeforeEach void beforeEach() { createTld("tld"); @@ -55,7 +51,7 @@ class GetDomainCommandTest extends CommandTestCase { persistActiveDomain("example.tld"); runCommand("example.tld", "--expand"); assertInStdout("fullyQualifiedDomainName=example.tld"); - assertInStdout("contactId=contact1234"); + assertInStdout("contact=Key(ContactResource(\"3-ROID\"))"); assertInStdout( "Websafe key: " + "kind:DomainBase" @@ -70,7 +66,7 @@ class GetDomainCommandTest extends CommandTestCase { persistActiveDomain("xn--aualito-txac.xn--q9jyb4c"); runCommand("çauçalito.みんな", "--expand"); assertInStdout("fullyQualifiedDomainName=xn--aualito-txac.xn--q9jyb4c"); - assertInStdout("contactId=contact1234"); + assertInStdout("contact=Key(ContactResource(\"4-ROID\"))"); } @Test @@ -94,15 +90,18 @@ class GetDomainCommandTest extends CommandTestCase { @Test void testSuccess_domainDeletedInFuture() throws Exception { - persistResource(newDomainBase("example.tld").asBuilder() - .setDeletionTime(now.plusDays(1)).build()); - runCommand("example.tld", "--read_timestamp=" + now.plusMonths(1)); + persistResource( + newDomainBase("example.tld") + .asBuilder() + .setDeletionTime(fakeClock.nowUtc().plusDays(1)) + .build()); + runCommand("example.tld", "--read_timestamp=" + fakeClock.nowUtc().plusMonths(1)); assertInStdout("Domain 'example.tld' does not exist or is deleted"); } @Test void testSuccess_deletedDomain() throws Exception { - persistDeletedDomain("example.tld", now.minusDays(1)); + persistDeletedDomain("example.tld", fakeClock.nowUtc().minusDays(1)); runCommand("example.tld"); assertInStdout("Domain 'example.tld' does not exist or is deleted"); } diff --git a/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java b/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java index fa0ac93ac..0782960a0 100644 --- a/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java +++ b/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java @@ -26,13 +26,20 @@ import static org.joda.time.DateTimeZone.UTC; import static org.junit.jupiter.api.Assertions.assertThrows; import com.beust.jcommander.ParameterException; +import google.registry.testing.TmOverrideExtension; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link GetResourceByKeyCommand}. */ class GetResourceByKeyCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private DateTime now = DateTime.now(UTC); @BeforeEach diff --git a/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java b/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java index f2426de0c..07c994991 100644 --- a/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java +++ b/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java @@ -20,6 +20,7 @@ import static google.registry.tools.LevelDbLogReader.HEADER_SIZE; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.EntityTranslator; import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.tools.LevelDbLogReader.ChunkType; import java.io.File; import java.io.FileNotFoundException; @@ -27,6 +28,7 @@ import java.io.FileOutputStream; import java.io.IOException; /** Utility class for building a leveldb logfile. */ +@DeleteAfterMigration public final class LevelDbFileBuilder { private final FileOutputStream out; diff --git a/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java b/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java index 04b5ed686..b66261f43 100644 --- a/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java +++ b/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java @@ -25,10 +25,12 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; import google.registry.model.contact.ContactResource; import google.registry.testing.AppEngineExtension; import google.registry.testing.DatabaseHelper; +import google.registry.testing.TmOverrideExtension; import google.registry.tools.EntityWrapper.Property; import java.io.File; import java.io.IOException; import java.nio.file.Path; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; @@ -40,6 +42,10 @@ public class LevelDbFileBuilderTest { @TempDir Path tmpDir; + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/tools/MutatingCommandTest.java b/core/src/test/java/google/registry/tools/MutatingCommandTest.java index 285d49d55..4417a216a 100644 --- a/core/src/test/java/google/registry/tools/MutatingCommandTest.java +++ b/core/src/test/java/google/registry/tools/MutatingCommandTest.java @@ -19,6 +19,7 @@ import static com.google.common.truth.Truth.assertThat; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.deleteResource; +import static google.registry.testing.DatabaseHelper.existsInDb; import static google.registry.testing.DatabaseHelper.loadByEntity; import static google.registry.testing.DatabaseHelper.persistActiveHost; import static google.registry.testing.DatabaseHelper.persistNewRegistrar; @@ -181,10 +182,10 @@ public class MutatingCommandTest { + registrar2 + "\n"); String results = command.execute(); assertThat(results).isEqualTo("Updated 4 entities.\n"); - assertThat(loadByEntity(host1)).isNull(); - assertThat(loadByEntity(host2)).isNull(); - assertThat(loadByEntity(registrar1)).isNull(); - assertThat(loadByEntity(registrar2)).isNull(); + assertThat(existsInDb(host1)).isFalse(); + assertThat(existsInDb(host2)).isFalse(); + assertThat(existsInDb(registrar1)).isFalse(); + assertThat(existsInDb(registrar2)).isFalse(); } @Test @@ -241,9 +242,9 @@ public class MutatingCommandTest { + "blockPremiumNames: false -> true\n"); String results = command.execute(); assertThat(results).isEqualTo("Updated 4 entities.\n"); - assertThat(loadByEntity(host1)).isNull(); + assertThat(existsInDb(host1)).isFalse(); assertThat(loadByEntity(host2)).isEqualTo(newHost2); - assertThat(loadByEntity(registrar1)).isNull(); + assertThat(existsInDb(registrar1)).isFalse(); assertThat(loadByEntity(registrar2)).isEqualTo(newRegistrar2); } @@ -282,7 +283,7 @@ public class MutatingCommandTest { IllegalStateException thrown = assertThrows(IllegalStateException.class, command::execute); assertThat(thrown).hasMessageThat().contains("Entity changed since init() was called."); - assertThat(loadByEntity(host1)).isNull(); + assertThat(existsInDb(host1)).isFalse(); assertThat(loadByEntity(host2)).isEqualTo(newHost2); // These two shouldn't've changed. assertThat(loadByEntity(registrar1)).isEqualTo(registrar1); diff --git a/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java b/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java index f9dd48551..ba5f1868c 100644 --- a/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java +++ b/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java @@ -23,6 +23,7 @@ import static google.registry.testing.DatabaseHelper.loadRegistrar; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DatabaseHelper.persistSimpleResource; import static google.registry.testing.DatabaseHelper.persistSimpleResources; +import static google.registry.testing.DatabaseHelper.putInDb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -334,8 +335,7 @@ class RegistrarContactCommandTest extends CommandTestCase { + + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + DomainBase domain; HistoryEntry historyEntry; diff --git a/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java b/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java index 77e037e7b..29de8082f 100644 --- a/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java +++ b/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java @@ -25,29 +25,20 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.beust.jcommander.ParameterException; import com.google.common.collect.ImmutableMap; import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; import google.registry.model.registrar.Registrar; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import google.registry.util.Clock; import java.util.List; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.shaded.com.google.common.collect.ImmutableList; /** Unit tests for {@link RenewDomainCommand}. */ public class RenewDomainCommandTest extends EppToolCommandTestCase { - @RegisterExtension public final InjectExtension inject = new InjectExtension(); - - private final Clock clock = new FakeClock(DateTime.parse("2015-04-05T05:05:05Z")); - @BeforeEach void beforeEach() { - inject.setStaticField(Ofy.class, "clock", clock); - command.clock = clock; + fakeClock.setTo(DateTime.parse("2015-04-05T05:05:05Z")); + command.clock = fakeClock; } @Test diff --git a/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java b/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java index 0e572eddf..6e86bb659 100644 --- a/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java +++ b/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java @@ -25,11 +25,18 @@ import google.registry.model.ImmutableObject; import google.registry.model.contact.ContactResource; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveEntitiesCommand}. */ class ResaveEntitiesCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @Test void testSuccess_createsCommitLogs() throws Exception { ContactResource contact1 = persistActiveContact("contact1"); diff --git a/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java b/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java index 156f3881b..a76e5e607 100644 --- a/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java +++ b/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java @@ -28,12 +28,19 @@ import google.registry.model.ofy.CommitLogMutation; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarContact; import google.registry.model.tld.Registry; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveEnvironmentEntitiesCommand}. */ class ResaveEnvironmentEntitiesCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @Test void testSuccess_noop() throws Exception { // Get rid of all the entities that this command runs on so that it does nothing. diff --git a/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java b/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java index 35fa8ddcc..0cc47d5f9 100644 --- a/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java +++ b/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java @@ -22,11 +22,18 @@ import google.registry.model.ImmutableObject; import google.registry.model.contact.ContactResource; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveEppResourceCommand}. */ class ResaveEppResourcesCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @Test void testSuccess_createsCommitLogs() throws Exception { ContactResource contact = persistActiveContact("contact"); diff --git a/core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java b/core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java deleted file mode 100644 index 12f75ec16..000000000 --- a/core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.beust.jcommander.ParameterException; -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.DualDatabaseTest; -import google.registry.testing.TestOfyAndSql; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; - -/** Tests for {@link SetDatabaseMigrationStateCommand}. */ -@DualDatabaseTest -public class SetDatabaseMigrationStateCommandTest - extends CommandTestCase { - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @TestOfyAndSql - void testSuccess_setsBasicSchedule() throws Exception { - assertThat(DatabaseMigrationStateSchedule.get()).isEqualTo(DEFAULT_TRANSITION_MAP); - assertThat(jpaTm().transact(() -> jpaTm().loadSingleton(DatabaseMigrationStateSchedule.class))) - .isEmpty(); - runCommandForced("--migration_schedule=1970-01-01T00:00:00.000Z=DATASTORE_ONLY"); - // use a raw ofy call to check what's in the DB - jpaTm() - .transact( - () -> - assertThat( - jpaTm() - .loadSingleton(DatabaseMigrationStateSchedule.class) - .get() - .migrationTransitions) - .isEqualTo(DEFAULT_TRANSITION_MAP)); - assertThat(DatabaseMigrationStateSchedule.get()).isEqualTo(DEFAULT_TRANSITION_MAP); - } - - @TestOfyAndSql - void testSuccess_fullSchedule() throws Exception { - DateTime now = fakeClock.nowUtc(); - DateTime datastorePrimary = now.plusHours(1); - DateTime datastorePrimaryNoAsync = now.plusHours(2); - DateTime datastorePrimaryReadOnly = now.plusHours(3); - DateTime sqlPrimary = now.plusHours(4); - DateTime sqlOnly = now.plusHours(5); - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY," - + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY," - + "%s=SQL_PRIMARY,%s=SQL_ONLY", - START_OF_TIME, - datastorePrimary, - datastorePrimaryNoAsync, - datastorePrimaryReadOnly, - sqlPrimary, - sqlOnly)); - assertThat(DatabaseMigrationStateSchedule.get().toValueMap()) - .containsExactlyEntriesIn( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - datastorePrimary, - MigrationState.DATASTORE_PRIMARY, - datastorePrimaryNoAsync, - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - datastorePrimaryReadOnly, - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - sqlPrimary, - MigrationState.SQL_PRIMARY, - sqlOnly, - MigrationState.SQL_ONLY)); - } - - @TestOfyAndSql - void testSuccess_warnsOnChangeSoon() throws Exception { - DateTime now = fakeClock.nowUtc(); - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY", - START_OF_TIME, now.plusMinutes(1))); - assertThat(DatabaseMigrationStateSchedule.get().toValueMap()) - .containsExactlyEntriesIn( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMinutes(1), - MigrationState.DATASTORE_PRIMARY)); - assertInStdout("MAY BE DANGEROUS"); - } - - @TestOfyAndSql - void testSuccess_goesBackward() throws Exception { - DateTime now = fakeClock.nowUtc(); - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY," - + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY," - + "%s=DATASTORE_PRIMARY", - START_OF_TIME, now.plusHours(1), now.plusHours(2), now.plusHours(3), now.plusHours(4))); - assertThat(DatabaseMigrationStateSchedule.get().toValueMap()) - .containsExactlyEntriesIn( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusHours(1), - MigrationState.DATASTORE_PRIMARY, - now.plusHours(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusHours(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - now.plusHours(4), - MigrationState.DATASTORE_PRIMARY)); - } - - @TestOfyAndSql - void testFailure_invalidTransition() { - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY_READ_ONLY", - START_OF_TIME, START_OF_TIME.plusHours(1)))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "validStateTransitions map cannot transition from DATASTORE_ONLY " - + "to DATASTORE_PRIMARY_READ_ONLY."); - } - - @TestOfyAndSql - void testFailure_invalidTransitionFromOldToNew() { - // The map we pass in is valid by itself, but we can't go from DATASTORE_ONLY now to - // DATASTORE_PRIMARY_READ_ONLY now - DateTime now = fakeClock.nowUtc(); - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY," - + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY", - START_OF_TIME, now.minusHours(3), now.minusHours(2), now.minusHours(1)))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "Cannot transition from current state-as-of-now DATASTORE_ONLY " - + "to new state-as-of-now DATASTORE_PRIMARY_READ_ONLY"); - } - - @TestOfyAndSql - void testFailure_invalidParams() { - assertThrows(ParameterException.class, this::runCommandForced); - assertThrows(ParameterException.class, () -> runCommandForced("--migration_schedule=FOOBAR")); - assertThrows( - ParameterException.class, - () -> runCommandForced("--migration_schedule=1970-01-01T00:00:00.000Z=FOOBAR")); - } -} diff --git a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java index 32e709bfb..8c43d8045 100644 --- a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java @@ -38,17 +38,24 @@ import google.registry.model.eppcommon.StatusValue; import google.registry.model.host.HostResource; import google.registry.persistence.VKey; import google.registry.testing.FakeClock; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import java.net.InetAddress; import java.util.Map; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Duration; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link GenerateZoneFilesAction}. */ class GenerateZoneFilesActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @Test diff --git a/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java b/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java index 082d968a2..f15fadc27 100644 --- a/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java +++ b/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java @@ -39,13 +39,20 @@ import google.registry.model.ofy.CommitLogCheckpointRoot; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; import google.registry.testing.FakeResponse; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import org.joda.time.DateTime; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link KillAllCommitLogsAction}. */ class KillAllCommitLogsActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final ImmutableList> AFFECTED_TYPES = ImmutableList.of( CommitLogBucket.class, diff --git a/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java b/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java index 31193c6c4..4827d9f22 100644 --- a/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java @@ -52,15 +52,22 @@ import google.registry.model.poll.PollMessage; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.DatabaseHelper; import google.registry.testing.FakeResponse; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import java.util.stream.Stream; import org.joda.money.CurrencyUnit; import org.joda.money.Money; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link KillAllEppResourcesAction}. */ class KillAllEppResourcesActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final ImmutableSet AFFECTED_KINDS = Stream.of( EppResourceIndex.class, diff --git a/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java b/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java index 1c310459f..92b35995d 100644 --- a/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java @@ -29,13 +29,20 @@ import google.registry.model.domain.DomainBase; import google.registry.model.domain.DomainHistory; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.FakeResponse; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveAllHistoryEntriesAction}. */ class ResaveAllHistoryEntriesActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); diff --git a/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java b/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java index 322f9dc7d..b6bfcea22 100644 --- a/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java +++ b/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java @@ -16,6 +16,7 @@ package google.registry.ui.server.registrar; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; +import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects; import static google.registry.testing.CertificateSamples.SAMPLE_CERT; import static google.registry.testing.CertificateSamples.SAMPLE_CERT2; import static google.registry.testing.CertificateSamples.SAMPLE_CERT2_HASH; @@ -48,13 +49,19 @@ class SecuritySettingsTest extends RegistrarSettingsActionTestCase { .asBuilder() .setClientCertificate(SAMPLE_CERT3, clock.nowUtc()) .build(); - Map response = action.handleJsonRequest(ImmutableMap.of( - "op", "update", - "id", CLIENT_ID, - "args", modified.toJsonMap())); + Map modifiedJsonMap = modified.toJsonMap(); + Map response = + action.handleJsonRequest( + ImmutableMap.of( + "op", "update", + "id", CLIENT_ID, + "args", modifiedJsonMap)); + modifiedJsonMap.put("lastUpdateTime", clock.nowUtc().toString()); assertThat(response).containsEntry("status", "SUCCESS"); - assertThat(response).containsEntry("results", ImmutableList.of(modified.toJsonMap())); - assertThat(loadRegistrar(CLIENT_ID)).isEqualTo(modified); + assertThat(response).containsEntry("results", ImmutableList.of(modifiedJsonMap)); + assertAboutImmutableObjects() + .that(loadRegistrar(CLIENT_ID)) + .isEqualExceptFields(modified, "lastUpdateTime"); assertMetric(CLIENT_ID, "update", "[OWNER]", "SUCCESS"); verifyNotificationEmailsSent(); } diff --git a/core/src/test/java/google/registry/webdriver/TestServerExtension.java b/core/src/test/java/google/registry/webdriver/TestServerExtension.java index 80b51d0df..67f570e10 100644 --- a/core/src/test/java/google/registry/webdriver/TestServerExtension.java +++ b/core/src/test/java/google/registry/webdriver/TestServerExtension.java @@ -30,8 +30,6 @@ import google.registry.server.Fixture; import google.registry.server.Route; import google.registry.server.TestServer; import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.FakeClock; import google.registry.testing.UserInfo; import java.net.URL; import java.net.UnknownHostException; @@ -43,8 +41,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingDeque; import javax.servlet.Filter; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -211,8 +207,6 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC // Clear the SQL database and set it as primary (we have to do this out of band because the // AppEngineExtension can't natively do it for us yet due to remaining ofy dependencies) new JpaTestExtensions.Builder().buildIntegrationTestExtension().beforeEach(null); - DatabaseHelper.setMigrationScheduleToSqlPrimary( - new FakeClock(DateTime.now(DateTimeZone.UTC))); // sleep a few millis to make sure we get to SQL-primary mode Thread.sleep(4); AppEngineExtension.loadInitialData(); diff --git a/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java b/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java index 28339f496..a017ea707 100644 --- a/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java +++ b/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java @@ -110,8 +110,8 @@ class RegistrarWhoisResponseTest { .setTypes(ImmutableSet.of(RegistrarContact.Type.TECH)) .setVisibleInWhoisAsTech(true) .build()); - persistSimpleResources(contacts); persistResource(registrar); + persistSimpleResources(contacts); RegistrarWhoisResponse registrarWhoisResponse = new RegistrarWhoisResponse(registrar, clock.nowUtc()); diff --git a/core/src/test/resources/google/registry/module/backend/backend_routing.txt b/core/src/test/resources/google/registry/module/backend/backend_routing.txt index 0aaa6f78b..7f2ebf4ed 100644 --- a/core/src/test/resources/google/registry/module/backend/backend_routing.txt +++ b/core/src/test/resources/google/registry/module/backend/backend_routing.txt @@ -30,7 +30,6 @@ PATH CLASS /_dr/task/rdeReport RdeReportAction POST n INTERNAL,API APP ADMIN /_dr/task/rdeStaging RdeStagingAction GET,POST n INTERNAL,API APP ADMIN /_dr/task/rdeUpload RdeUploadAction POST n INTERNAL,API APP ADMIN -/_dr/task/refreshDnsOnHostRename RefreshDnsOnHostRenameAction GET n INTERNAL,API APP ADMIN /_dr/task/relockDomain RelockDomainAction POST y INTERNAL,API APP ADMIN /_dr/task/resaveAllEppResources ResaveAllEppResourcesAction GET n INTERNAL,API APP ADMIN /_dr/task/resaveAllEppResourcesPipeline ResaveAllEppResourcesPipelineAction GET n INTERNAL,API APP ADMIN diff --git a/docs/flows.md b/docs/flows.md index d05f2fd46..5a9f2cb17 100644 --- a/docs/flows.md +++ b/docs/flows.md @@ -31,8 +31,6 @@ An EPP flow that creates a new contact. * Resource with this id already exists. * 2306 * Declining contact disclosure is disallowed by server policy. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactDeleteFlow @@ -59,8 +57,6 @@ or failure message when the process is complete. * Resource status prohibits this operation. * 2305 * Resource to be deleted has active incoming references. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactInfoFlow @@ -107,8 +103,6 @@ transfer request, which then becomes effective immediately. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactTransferCancelFlow @@ -134,8 +128,6 @@ request. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactTransferQueryFlow @@ -186,8 +178,6 @@ that window, this flow allows the losing client to reject the transfer request. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactTransferRequestFlow @@ -216,8 +206,6 @@ or rejected, and the gaining registrar can also cancel the transfer request. * Resource with this id does not exist. * 2304 * Resource status prohibits this operation. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactUpdateFlow @@ -244,8 +232,6 @@ An EPP flow that updates a contact. * 2306 * Cannot add and remove the same value. * Declining contact disclosure is disallowed by server policy. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainCheckFlow @@ -410,8 +396,6 @@ An EPP flow that creates a new domain resource. * Too many nameservers set on this domain. * Domain labels cannot end with a dash. * Only encoded signed marks are supported. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainDeleteFlow @@ -436,8 +420,6 @@ An EPP flow that deletes a domain. * Resource status prohibits this operation. * 2305 * Domain to be deleted has subordinate hosts. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainInfoFlow @@ -514,8 +496,6 @@ comes in at the exact millisecond that the domain would have expired. * 2306 * Periods for domain registrations must be specified in years. * The requested fees cannot be provided in the requested currency. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainRestoreRequestFlow @@ -577,8 +557,6 @@ regardless of what the original expiration time was. * Domain is not eligible for restore. * 2306 * The requested fees cannot be provided in the requested currency. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferApproveFlow @@ -610,8 +588,6 @@ replaced with new ones with the correct approval time. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferCancelFlow @@ -642,8 +618,6 @@ transfer period passed. In this flow, those speculative objects are deleted. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferQueryFlow @@ -699,8 +673,6 @@ transfer period passed. In this flow, those speculative objects are deleted. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferRequestFlow @@ -762,8 +734,6 @@ new ones with the correct approval time). EPP extension. * Periods for domain registrations must be specified in years. * The requested fees cannot be provided in the requested currency. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainUpdateFlow @@ -822,8 +792,6 @@ statuses are updated at once. * The secDNS:all element must have value 'true' if present. * Too many DS records set on a domain. * Too many nameservers set on this domain. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## HelloFlow @@ -886,8 +854,6 @@ allows creating a host name, and if necessary enqueues tasks to update DNS. * Superordinate domain for this hostname is in pending delete. * 2306 * Host names must be at least two levels below the registry suffix. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## HostDeleteFlow @@ -918,8 +884,6 @@ or failure message when the process is complete. * Resource status prohibits this operation. * 2305 * Resource to be deleted has active incoming references. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## HostInfoFlow @@ -992,8 +956,6 @@ are enqueued to update DNS accordingly. * 2306 * Cannot add and remove the same value. * Host names must be at least two levels below the registry suffix. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## LoginFlow diff --git a/release/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml index df2854309..61b0c6b74 100644 --- a/release/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -88,8 +88,6 @@ steps: beam_pipeline_common \ ${TAG_NAME} \ ${PROJECT_ID} \ - google.registry.beam.initsql.InitSqlPipeline \ - google/registry/beam/init_sql_pipeline_metadata.json \ google.registry.beam.datastore.BulkDeleteDatastorePipeline \ google/registry/beam/bulk_delete_datastore_pipeline_metadata.json \ google.registry.beam.spec11.Spec11Pipeline \