From 2f8be045c7365d46f78104e89cc45ae9b5fc2648 Mon Sep 17 00:00:00 2001 From: gbrodman Date: Mon, 13 Jun 2022 15:10:35 -0400 Subject: [PATCH] Delete code relating to SQL init and scheduling (#1661) One of the more significant changes introduced in this PR is that we use SQL as the backing database in all tests unless otherwise specified, e.g. by using the TmOverrideExtension. We change various ofy-related tests to use this. This includes various changes: - Deletion of SqlEntity/DatastoreEntity and related classes. Includes any necessary changes because of that (e.g. getting a nice SQL key on error in RegistryJpaIO). - Deletion of classes that used libraries from the init-sql code (RefreshDnsOnHostRenameAction) - Removal of the JpaTransactionManager's backup implementation - Modification of RegistryJpaWriteTest to not use init-sql code - Removal of the Transaction class and related classes, however it does not remove the TransactionEntity class as that would require DB changes - Removal of anything related to the actual usage of the database migration schedule or read-only phases - Various test changes and fixes to account for the differences in SQL (like how foreign keys need to exist) This deliberately doesn't do anything to alter the objects actually stored in the DB yet, just how we use them --- core/build.gradle | 47 -- .../registry/backup/CommitLogImports.java | 140 ----- .../batch/RefreshDnsOnHostRenameAction.java | 379 -------------- .../registry/beam/common/RegistryJpaIO.java | 24 +- .../registry/beam/initsql/BackupPaths.java | 119 ----- .../registry/beam/initsql/DomainBaseUtil.java | 78 --- .../beam/initsql/InitSqlPipeline.java | 242 --------- .../beam/initsql/InitSqlPipelineOptions.java | 47 -- .../google/registry/beam/initsql/README.md | 17 - .../registry/beam/initsql/Transforms.java | 485 ------------------ .../beam/invoicing/InvoicingPipeline.java | 21 +- .../invoicing/InvoicingPipelineOptions.java | 5 - .../registry/beam/spec11/DomainNameInfo.java | 35 +- .../registry/beam/spec11/Spec11Pipeline.java | 27 +- .../beam/spec11/Spec11PipelineOptions.java | 5 - .../google/registry/flows/EppException.java | 9 - .../google/registry/flows/FlowRunner.java | 4 - .../flows/contact/ContactCreateFlow.java | 1 - .../flows/contact/ContactDeleteFlow.java | 3 - .../contact/ContactTransferApproveFlow.java | 1 - .../contact/ContactTransferCancelFlow.java | 1 - .../contact/ContactTransferRejectFlow.java | 1 - .../contact/ContactTransferRequestFlow.java | 1 - .../flows/contact/ContactUpdateFlow.java | 1 - .../flows/domain/DomainCreateFlow.java | 1 - .../flows/domain/DomainDeleteFlow.java | 1 - .../flows/domain/DomainRenewFlow.java | 1 - .../domain/DomainRestoreRequestFlow.java | 1 - .../domain/DomainTransferApproveFlow.java | 1 - .../domain/DomainTransferCancelFlow.java | 1 - .../domain/DomainTransferRejectFlow.java | 1 - .../domain/DomainTransferRequestFlow.java | 1 - .../flows/domain/DomainUpdateFlow.java | 1 - .../registry/flows/host/HostCreateFlow.java | 1 - .../registry/flows/host/HostDeleteFlow.java | 3 - .../registry/flows/host/HostUpdateFlow.java | 5 - .../mapreduce/inputs/EppResourceInputs.java | 2 + .../registry/model/billing/BillingEvent.java | 10 +- .../model/bulkquery/DomainBaseLite.java | 3 +- .../model/bulkquery/DomainHistoryHost.java | 3 +- .../model/bulkquery/DomainHistoryLite.java | 3 +- .../registry/model/bulkquery/DomainHost.java | 3 +- .../google/registry/model/common/Cursor.java | 3 +- .../DatabaseMigrationStateSchedule.java | 5 +- .../model/common/EntityGroupRoot.java | 3 +- .../model/contact/ContactHistory.java | 10 +- .../model/contact/ContactResource.java | 4 +- .../registry/model/domain/DomainBase.java | 4 +- .../registry/model/domain/DomainHistory.java | 10 +- .../registry/model/domain/GracePeriod.java | 6 +- .../registry/model/domain/RegistryLock.java | 3 +- .../domain/secdns/DomainDsDataHistory.java | 4 +- .../model/domain/token/AllocationToken.java | 3 +- .../registry/model/host/HostHistory.java | 10 +- .../registry/model/host/HostResource.java | 4 +- .../model/index/EppResourceIndex.java | 3 +- .../model/index/EppResourceIndexBucket.java | 3 +- .../registry/model/index/ForeignKeyIndex.java | 10 +- .../registry/model/ofy/CommitLogBucket.java | 3 +- .../model/ofy/CommitLogCheckpoint.java | 3 +- .../model/ofy/CommitLogCheckpointRoot.java | 3 +- .../registry/model/ofy/CommitLogManifest.java | 3 +- .../registry/model/ofy/CommitLogMutation.java | 3 +- .../ofy/DatastoreTransactionManager.java | 50 +- .../java/google/registry/model/ofy/Ofy.java | 5 - .../registry/model/poll/PollMessage.java | 3 +- .../registry/model/rde/RdeRevision.java | 3 +- .../registry/model/registrar/Registrar.java | 3 +- .../model/registrar/RegistrarContact.java | 4 +- .../model/replay/DatastoreAndSqlEntity.java | 33 -- .../model/replay/DatastoreEntity.java | 32 -- .../model/replay/DatastoreOnlyEntity.java | 27 - .../model/replay/LastSqlTransaction.java | 2 +- .../model/replay/NonReplicatedEntity.java | 37 -- .../registry/model/replay/ReplayGap.java | 2 +- .../registry/model/replay/SqlEntity.java | 48 -- .../registry/model/replay/SqlOnlyEntity.java | 27 - .../model/replay/SqlReplayCheckpoint.java | 4 +- .../reporting/DomainTransactionRecord.java | 3 +- .../model/reporting/HistoryEntry.java | 11 +- .../model/reporting/Spec11ThreatMatch.java | 3 +- .../google/registry/model/server/Lock.java | 7 +- .../registry/model/server/ServerSecret.java | 3 +- .../model/smd/SignedMarkRevocationList.java | 3 +- .../google/registry/model/tld/Registry.java | 4 +- .../registry/model/tld/label/PremiumList.java | 6 +- .../model/tld/label/ReservedList.java | 8 +- .../registry/model/tmch/ClaimsEntry.java | 3 +- .../registry/model/tmch/ClaimsList.java | 3 +- .../google/registry/model/tmch/TmchCrl.java | 3 +- .../CommitLogRevisionsTranslatorFactory.java | 7 - .../UpdateAutoTimestampTranslatorFactory.java | 16 - .../backend/BackendRequestComponent.java | 3 - .../JpaTransactionManagerImpl.java | 170 +----- .../ReadOnlyCheckingEntityManager.java | 322 ------------ .../transaction/ReadOnlyCheckingQuery.java | 211 -------- .../ReadOnlyCheckingTypedQuery.java | 208 -------- .../persistence/transaction/Transaction.java | 336 ------------ .../transaction/TransactionEntity.java | 5 +- .../transaction/TransactionManager.java | 6 - .../TransactionManagerFactory.java | 59 +-- .../google/registry/rde/RdeStagingMapper.java | 2 + .../registry/rde/RdeStagingReducer.java | 2 + .../billing/GenerateInvoicesAction.java | 13 - .../spec11/GenerateSpec11ReportAction.java | 11 - .../registry/request/RequestModule.java | 12 - .../registry/request/RequestParameters.java | 8 - .../DedupeOneTimeBillingEventIdsCommand.java | 88 ---- .../GetDatabaseMigrationStateCommand.java | 36 -- .../registry/tools/ListTxnsCommand.java | 116 ----- .../registry/tools/MutatingCommand.java | 14 +- .../google/registry/tools/RegistryTool.java | 5 - .../tools/RemoveRegistryOneKeyCommand.java | 2 + .../registry/tools/ReplayTxnsCommand.java | 59 --- .../ResaveEnvironmentEntitiesCommand.java | 2 + .../SetDatabaseMigrationStateCommand.java | 72 --- .../tools/params/TransitionListParameter.java | 9 - .../beam/init_sql_pipeline_metadata.json | 68 --- .../beam/invoicing_pipeline_metadata.json | 8 - .../beam/spec11_pipeline_metadata.json | 8 - .../validate_database_pipeline_metadata.json | 48 -- .../backup/CommitLogCheckpointActionTest.java | 6 + .../backup/DeleteOldCommitLogsActionTest.java | 6 + .../backup/ExportCommitLogDiffActionTest.java | 6 + .../RefreshDnsOnHostRenameActionTest.java | 279 ---------- .../ResaveAllEppResourcesActionTest.java | 7 + .../beam/common/RegistryJpaWriteTest.java | 65 +-- .../beam/initsql/BackupPathsTest.java | 63 --- .../beam/initsql/BackupTestStore.java | 204 -------- .../beam/initsql/BackupTestStoreTest.java | 242 --------- .../beam/initsql/CommitLogTransformsTest.java | 245 --------- .../beam/initsql/DatastoreSetupHelper.java | 295 ----------- .../beam/initsql/DomainBaseUtilTest.java | 233 --------- .../initsql/ExportLoadingTransformsTest.java | 201 -------- .../initsql/InitSqlPipelineGraphTest.java | 67 --- .../initsql/InitSqlPipelineOptionsTest.java | 27 - .../beam/initsql/InitSqlPipelineTest.java | 136 ----- .../beam/initsql/InitSqlTestUtils.java | 157 ------ .../initsql/LoadDatastoreSnapshotTest.java | 190 ------- .../registry/beam/initsql/TransformsTest.java | 64 --- .../beam/invoicing/InvoicingPipelineTest.java | 6 - .../registry/beam/rde/RdePipelineTest.java | 5 - .../ResaveAllEppResourcesPipelineTest.java | 5 - .../beam/spec11/Spec11PipelineTest.java | 7 - .../flows/contact/ContactCreateFlowTest.java | 11 - .../flows/contact/ContactDeleteFlowTest.java | 21 - .../ContactTransferApproveFlowTest.java | 11 - .../ContactTransferCancelFlowTest.java | 11 - .../ContactTransferRejectFlowTest.java | 11 - .../ContactTransferRequestFlowTest.java | 11 - .../flows/contact/ContactUpdateFlowTest.java | 12 - .../flows/domain/DomainCreateFlowTest.java | 21 - .../flows/domain/DomainDeleteFlowTest.java | 12 - .../flows/domain/DomainRenewFlowTest.java | 19 - .../domain/DomainRestoreRequestFlowTest.java | 12 - .../domain/DomainTransferApproveFlowTest.java | 11 - .../domain/DomainTransferCancelFlowTest.java | 11 - .../domain/DomainTransferRejectFlowTest.java | 11 - .../domain/DomainTransferRequestFlowTest.java | 14 - .../flows/domain/DomainUpdateFlowTest.java | 13 - .../flows/host/HostCreateFlowTest.java | 11 - .../flows/host/HostDeleteFlowTest.java | 21 - .../flows/host/HostUpdateFlowTest.java | 49 -- .../flows/session/LoginFlowTestCase.java | 4 +- .../inputs/ChildEntityInputTest.java | 6 + .../inputs/CommitLogManifestInputTest.java | 6 + .../inputs/EppResourceInputsTest.java | 6 + .../model/CreateAutoTimestampTest.java | 2 - .../registry/model/ImmutableObjectTest.java | 34 +- .../model/UpdateAutoTimestampTest.java | 2 - .../DatabaseMigrationStateScheduleTest.java | 217 -------- .../registry/model/domain/DomainBaseTest.java | 166 +++--- .../model/index/EppResourceIndexTest.java | 7 + .../model/ofy/CommitLogBucketTest.java | 6 + .../ofy/DatastoreTransactionManagerTest.java | 8 +- .../registry/model/ofy/OfyCommitLogTest.java | 6 + .../google/registry/model/ofy/OfyTest.java | 17 +- .../registry/model/replay/EntityTest.java | 152 ------ ...mmitLogRevisionsTranslatorFactoryTest.java | 8 +- .../registry/persistence/BillingVKeyTest.java | 2 - .../persistence/DomainHistoryVKeyTest.java | 2 - .../EntityCallbacksListenerTest.java | 3 +- .../converter/BloomFilterConverterTest.java | 2 - .../converter/CurrencyUnitConverterTest.java | 2 - .../converter/DurationConverterTest.java | 2 - .../InetAddressSetConverterTest.java | 2 - .../converter/JodaMoneyConverterTest.java | 3 - .../converter/LocalDateConverterTest.java | 2 - .../JpaTransactionManagerExtension.java | 29 +- .../JpaTransactionManagerImplTest.java | 6 - ...eplicaSimulatingJpaTransactionManager.java | 10 - .../transaction/TransactionManagerTest.java | 12 +- .../transaction/TransactionTest.java | 189 ------- .../rde/RdeStagingActionDatastoreTest.java | 6 + .../registry/rde/RdeStagingMapperTest.java | 6 + .../registry/rde/RdeStagingReducerTest.java | 6 + .../rde/RegistrarToXjcConverterTest.java | 11 +- .../billing/GenerateInvoicesActionTest.java | 4 - .../GenerateSpec11ReportActionTest.java | 4 - .../registrar/RegistrarContactTest.java | 5 - .../schema/registrar/RegistrarDaoTest.java | 5 - .../registry/schema/replay/SqlEntityTest.java | 70 --- .../registry/testing/DatabaseHelper.java | 120 +---- .../google/registry/testing/TestObject.java | 6 +- .../registry/testing/TmOverrideExtension.java | 26 +- .../registry/tmch/NordnUploadActionTest.java | 11 +- .../registry/tools/CompareDbBackupsTest.java | 2 + .../registry/tools/CreateTldCommandTest.java | 4 +- ...dupeOneTimeBillingEventIdsCommandTest.java | 151 ------ .../GetDatabaseMigrationStateCommandTest.java | 68 --- .../registry/tools/GetDomainCommandTest.java | 19 +- .../tools/GetResourceByKeyCommandTest.java | 7 + .../registry/tools/LevelDbFileBuilder.java | 2 + .../tools/LevelDbFileBuilderTest.java | 6 + .../registry/tools/MutatingCommandTest.java | 15 +- .../tools/RegistrarContactCommandTest.java | 4 +- .../RemoveRegistryOneKeyCommandTest.java | 8 + .../tools/RenewDomainCommandTest.java | 13 +- .../tools/ResaveEntitiesCommandTest.java | 7 + .../ResaveEnvironmentEntitiesCommandTest.java | 7 + .../tools/ResaveEppResourcesCommandTest.java | 7 + .../SetDatabaseMigrationStateCommandTest.java | 186 ------- .../server/GenerateZoneFilesActionTest.java | 7 + .../server/KillAllCommitLogsActionTest.java | 7 + .../server/KillAllEppResourcesActionTest.java | 7 + .../ResaveAllHistoryEntriesActionTest.java | 7 + .../registrar/SecuritySettingsTest.java | 19 +- .../webdriver/TestServerExtension.java | 6 - .../whois/RegistrarWhoisResponseTest.java | 2 +- .../module/backend/backend_routing.txt | 1 - docs/flows.md | 38 -- release/cloudbuild-nomulus.yaml | 2 - 232 files changed, 457 insertions(+), 8245 deletions(-) delete mode 100644 core/src/main/java/google/registry/backup/CommitLogImports.java delete mode 100644 core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java delete mode 100644 core/src/main/java/google/registry/beam/initsql/BackupPaths.java delete mode 100644 core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java delete mode 100644 core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java delete mode 100644 core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java delete mode 100644 core/src/main/java/google/registry/beam/initsql/README.md delete mode 100644 core/src/main/java/google/registry/beam/initsql/Transforms.java delete mode 100644 core/src/main/java/google/registry/model/replay/DatastoreAndSqlEntity.java delete mode 100644 core/src/main/java/google/registry/model/replay/DatastoreEntity.java delete mode 100644 core/src/main/java/google/registry/model/replay/DatastoreOnlyEntity.java delete mode 100644 core/src/main/java/google/registry/model/replay/NonReplicatedEntity.java delete mode 100644 core/src/main/java/google/registry/model/replay/SqlEntity.java delete mode 100644 core/src/main/java/google/registry/model/replay/SqlOnlyEntity.java delete mode 100644 core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java delete mode 100644 core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingQuery.java delete mode 100644 core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingTypedQuery.java delete mode 100644 core/src/main/java/google/registry/persistence/transaction/Transaction.java delete mode 100644 core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java delete mode 100644 core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java delete mode 100644 core/src/main/java/google/registry/tools/ListTxnsCommand.java delete mode 100644 core/src/main/java/google/registry/tools/ReplayTxnsCommand.java delete mode 100644 core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java delete mode 100644 core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json delete mode 100644 core/src/main/resources/google/registry/beam/validate_database_pipeline_metadata.json delete mode 100644 core/src/test/java/google/registry/batch/RefreshDnsOnHostRenameActionTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/BackupPathsTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/BackupTestStore.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/DatastoreSetupHelper.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/ExportLoadingTransformsTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/InitSqlPipelineGraphTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/InitSqlPipelineOptionsTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/InitSqlTestUtils.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/LoadDatastoreSnapshotTest.java delete mode 100644 core/src/test/java/google/registry/beam/initsql/TransformsTest.java delete mode 100644 core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java delete mode 100644 core/src/test/java/google/registry/model/replay/EntityTest.java delete mode 100644 core/src/test/java/google/registry/persistence/transaction/TransactionTest.java delete mode 100644 core/src/test/java/google/registry/schema/replay/SqlEntityTest.java delete mode 100644 core/src/test/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommandTest.java delete mode 100644 core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java delete mode 100644 core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java diff --git a/core/build.gradle b/core/build.gradle index f918c353d..79f65dc0e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -765,11 +765,6 @@ createUberJar( // User should install gcloud and login to GCP before invoking this tasks. if (environment == 'alpha') { def pipelines = [ - initSql : - [ - mainClass: 'google.registry.beam.initsql.InitSqlPipeline', - metaData : 'google/registry/beam/init_sql_pipeline_metadata.json' - ], bulkDeleteDatastore: [ mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline', @@ -888,48 +883,6 @@ task buildToolImage(dependsOn: nomulus, type: Exec) { commandLine 'docker', 'build', '-t', 'nomulus-tool', '.' } -task generateInitSqlPipelineGraph(type: FilteringTest) { - tests = ['InitSqlPipelineGraphTest.createPipeline_compareGraph'] - ignoreFailures = true -} - -task updateInitSqlPipelineGraph(type: Copy) { - def graphRelativePath = 'google/registry/beam/initsql/' - from ("${projectDir}/build/resources/test/${graphRelativePath}") { - include 'pipeline_curr.dot' - rename 'curr', 'golden' - } - into "src/test/resources/${graphRelativePath}" - - dependsOn generateInitSqlPipelineGraph - - doLast { - if (com.google.common.base.Strings.isNullOrEmpty(project.dot_path)) { - getLogger().info('Property dot_path is null. Not creating image for pipeline graph.') - } - def dotPath = project.dot_path - if (!new File(dotPath).exists()) { - throw new RuntimeException( - """\ - ${dotPath} not found. Make sure graphviz is installed - and the dot_path property is set correctly.""" - .stripIndent()) - } - def goldenGraph = "src/test/resources/${graphRelativePath}/pipeline_golden.dot" - def goldenImage = "src/test/resources/${graphRelativePath}/pipeline_golden.png" - def cmd = "${dotPath} -Tpng -o \"${goldenImage}\" \"${goldenGraph}\"" - try { - rootProject.ext.execInBash(cmd, projectDir) - } catch (Throwable throwable) { - throw new RuntimeException( - """\ - Failed to generate golden image with command ${cmd} - Error: ${throwable.getMessage()} - """) - } - } -} - // Build the devtool jar. createUberJar( 'devtool', diff --git a/core/src/main/java/google/registry/backup/CommitLogImports.java b/core/src/main/java/google/registry/backup/CommitLogImports.java deleted file mode 100644 index 9c98cdee6..000000000 --- a/core/src/main/java/google/registry/backup/CommitLogImports.java +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.backup; - -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static google.registry.backup.BackupUtils.createDeserializingIterator; - -import com.google.common.collect.ImmutableList; -import google.registry.model.ImmutableObject; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.ofy.CommitLogCheckpoint; -import google.registry.model.ofy.CommitLogManifest; -import google.registry.model.ofy.CommitLogMutation; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.util.Iterator; - -/** - * Helpers for reading CommitLog records from a file. - * - *

This class is adapted from {@link RestoreCommitLogsAction}, and will be used in the initial - * population of the Cloud SQL database. - */ -@DeleteAfterMigration -public final class CommitLogImports { - - private CommitLogImports() {} - - /** - * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link - * ImmutableList} of {@link ImmutableList}s of {@link VersionedEntity} records where the inner - * lists each consist of one transaction. Upon completion the {@code inputStream} is closed. - * - *

The returned list may be empty, since CommitLogs are written at fixed intervals regardless - * if actual changes exist. Each sublist, however, will not be empty. - * - *

A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated) - * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence - * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted - * entity keys, whereas each {@code CommitLogMutation} contains one whole entity. - */ - static ImmutableList> loadEntitiesByTransaction( - InputStream inputStream) { - try (InputStream input = new BufferedInputStream(inputStream)) { - Iterator commitLogs = createDeserializingIterator(input, false); - checkState(commitLogs.hasNext()); - checkState(commitLogs.next() instanceof CommitLogCheckpoint); - - ImmutableList.Builder> resultBuilder = - new ImmutableList.Builder<>(); - ImmutableList.Builder currentTransactionBuilder = - new ImmutableList.Builder<>(); - - while (commitLogs.hasNext()) { - ImmutableObject currentObject = commitLogs.next(); - if (currentObject instanceof CommitLogManifest) { - // CommitLogManifest means we are starting a new transaction - addIfNonempty(resultBuilder, currentTransactionBuilder); - currentTransactionBuilder = new ImmutableList.Builder<>(); - VersionedEntity.fromManifest((CommitLogManifest) currentObject) - .forEach(currentTransactionBuilder::add); - } else if (currentObject instanceof CommitLogMutation) { - currentTransactionBuilder.add( - VersionedEntity.fromMutation((CommitLogMutation) currentObject)); - } else { - throw new IllegalStateException( - String.format("Unknown entity type %s in commit logs", currentObject.getClass())); - } - } - // Add the last transaction in (if it's not empty) - addIfNonempty(resultBuilder, currentTransactionBuilder); - return resultBuilder.build(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link - * ImmutableList} of {@link VersionedEntity} records. Upon completion the {@code inputStream} is - * closed. - * - *

The returned list may be empty, since CommitLogs are written at fixed intervals regardless - * if actual changes exist. - * - *

A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated) - * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence - * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted - * entity keys, whereas each {@code CommitLogMutation} contains one whole entity. - */ - static ImmutableList loadEntities(InputStream inputStream) { - return loadEntitiesByTransaction(inputStream).stream() - .flatMap(ImmutableList::stream) - .collect(toImmutableList()); - } - - /** Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link File}. */ - public static ImmutableList loadEntities(File commitLogFile) { - try { - return loadEntities(new FileInputStream(commitLogFile)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link - * ReadableByteChannel}. - */ - public static ImmutableList loadEntities(ReadableByteChannel channel) { - return loadEntities(Channels.newInputStream(channel)); - } - - private static void addIfNonempty( - ImmutableList.Builder> resultBuilder, - ImmutableList.Builder currentTransactionBuilder) { - ImmutableList currentTransaction = currentTransactionBuilder.build(); - if (!currentTransaction.isEmpty()) { - resultBuilder.add(currentTransaction); - } - } -} diff --git a/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java b/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java deleted file mode 100644 index 947250076..000000000 --- a/core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2017 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.batch; - -import static com.google.appengine.api.taskqueue.QueueConstants.maxLeaseCount; -import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static google.registry.batch.AsyncTaskEnqueuer.PARAM_HOST_KEY; -import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME; -import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME; -import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH; -import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput; -import static google.registry.model.EppResourceUtils.getLinkedDomainKeys; -import static google.registry.model.EppResourceUtils.isActive; -import static google.registry.model.EppResourceUtils.isDeleted; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.util.DateTimeUtils.latestOf; -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.logging.Level.INFO; -import static java.util.logging.Level.SEVERE; -import static org.joda.time.Duration.standardHours; - -import com.google.appengine.api.taskqueue.LeaseOptions; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.TaskHandle; -import com.google.appengine.api.taskqueue.TransientFailureException; -import com.google.appengine.tools.mapreduce.Mapper; -import com.google.appengine.tools.mapreduce.Reducer; -import com.google.appengine.tools.mapreduce.ReducerInput; -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.flogger.FluentLogger; -import google.registry.batch.AsyncTaskMetrics.OperationResult; -import google.registry.dns.DnsQueue; -import google.registry.mapreduce.MapreduceRunner; -import google.registry.mapreduce.inputs.NullInput; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.domain.DomainBase; -import google.registry.model.host.HostResource; -import google.registry.model.server.Lock; -import google.registry.persistence.VKey; -import google.registry.request.Action; -import google.registry.request.Response; -import google.registry.request.auth.Auth; -import google.registry.util.Clock; -import google.registry.util.NonFinalForTesting; -import google.registry.util.RequestStatusChecker; -import google.registry.util.Retrier; -import google.registry.util.SystemClock; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.logging.Level; -import javax.annotation.Nullable; -import javax.inject.Inject; -import javax.inject.Named; -import org.apache.http.HttpStatus; -import org.joda.time.DateTime; -import org.joda.time.Duration; - -/** Performs batched DNS refreshes for applicable domains following a host rename. */ -@Action( - service = Action.Service.BACKEND, - path = "/_dr/task/refreshDnsOnHostRename", - auth = Auth.AUTH_INTERNAL_OR_ADMIN) -@DeleteAfterMigration -@Deprecated -public class RefreshDnsOnHostRenameAction implements Runnable { - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final Duration LEASE_LENGTH = standardHours(4); - - @Inject AsyncTaskMetrics asyncTaskMetrics; - @Inject Clock clock; - @Inject MapreduceRunner mrRunner; - @Inject @Named(QUEUE_ASYNC_HOST_RENAME) Queue pullQueue; - - @Inject DnsQueue dnsQueue; - @Inject RequestStatusChecker requestStatusChecker; - @Inject Response response; - @Inject Retrier retrier; - @Inject RefreshDnsOnHostRenameAction() {} - - @Override - public void run() { - // Check if the lock can be acquired, and if not, a previous run of this mapreduce is still - // executing, so return early. - Optional lock = - Lock.acquire( - RefreshDnsOnHostRenameAction.class.getSimpleName(), - null, - LEASE_LENGTH, - requestStatusChecker, - false); - - if (!lock.isPresent()) { - logRespondAndUnlock(INFO, "Can't acquire lock; aborting.", lock); - return; - } - - // Lease the async tasks to process. - LeaseOptions options = - LeaseOptions.Builder.withCountLimit(maxLeaseCount()) - .leasePeriod(LEASE_LENGTH.getStandardSeconds(), SECONDS); - List tasks = pullQueue.leaseTasks(options); - asyncTaskMetrics.recordDnsRefreshBatchSize(tasks.size()); - - // Check if there are no tasks to process, and if so, return early. - if (tasks.isEmpty()) { - logRespondAndUnlock( - INFO, "No DNS refresh on host rename tasks to process in pull queue; finishing.", lock); - return; - } - - ImmutableList.Builder requestsBuilder = new ImmutableList.Builder<>(); - ImmutableList.Builder> hostKeys = new ImmutableList.Builder<>(); - final List requestsToDelete = new ArrayList<>(); - - for (TaskHandle task : tasks) { - try { - DnsRefreshRequest request = DnsRefreshRequest.createFromTask(task, clock.nowUtc()); - if (request.isRefreshNeeded()) { - requestsBuilder.add(request); - hostKeys.add(request.hostKey()); - } else { - // Skip hosts that are deleted. - requestsToDelete.add(request); - } - } catch (Exception e) { - logger.atSevere().withCause(e).log( - "Could not parse DNS refresh for host request, delaying task for a day: %s", task); - // Grab the lease for a whole day, so it won't continue throwing errors every five minutes. - pullQueue.modifyTaskLease(task, 1L, DAYS); - } - } - - deleteTasksWithRetry( - requestsToDelete, pullQueue, asyncTaskMetrics, retrier, OperationResult.STALE); - ImmutableList refreshRequests = requestsBuilder.build(); - if (refreshRequests.isEmpty()) { - logRespondAndUnlock( - INFO, "No async DNS refreshes to process because all renamed hosts are deleted.", lock); - } else { - logger.atInfo().log( - "Processing asynchronous DNS refresh for renamed hosts: %s", hostKeys.build()); - if (tm().isOfy()) { - runMapreduce(refreshRequests, lock); - } else { - try { - refreshRequests.stream() - .flatMap( - request -> - getLinkedDomainKeys(request.hostKey(), request.lastUpdateTime(), null) - .stream()) - .distinct() - .map(domainKey -> tm().transact(() -> tm().loadByKey(domainKey).getDomainName())) - .forEach( - domainName -> { - retrier.callWithRetry( - () -> dnsQueue.addDomainRefreshTask(domainName), - TransientFailureException.class); - logger.atInfo().log("Enqueued DNS refresh for domain '%s'.", domainName); - }); - deleteTasksWithRetry( - refreshRequests, - getQueue(QUEUE_ASYNC_HOST_RENAME), - asyncTaskMetrics, - retrier, - OperationResult.SUCCESS); - } catch (Throwable t) { - String message = "Error refreshing DNS on host rename."; - logger.atSevere().withCause(t).log(message); - response.setPayload(message); - response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); - } finally { - lock.get().release(); - } - } - } - } - - private void runMapreduce(ImmutableList refreshRequests, Optional lock) { - try { - mrRunner - .setJobName("Enqueue DNS refreshes for domains referencing renamed hosts") - .setModuleName("backend") - .setDefaultReduceShards(1) - .runMapreduce( - new RefreshDnsOnHostRenameMapper(refreshRequests, retrier), - new RefreshDnsOnHostRenameReducer(refreshRequests, lock.get(), retrier), - // Add an extra NullInput so that the reducer always fires exactly once. - ImmutableList.of(new NullInput<>(), createEntityInput(DomainBase.class))) - .sendLinkToMapreduceConsole(response); - } catch (Throwable t) { - logRespondAndUnlock( - SEVERE, "Error starting mapreduce to refresh DNS for renamed hosts.", lock); - } - } - - private void logRespondAndUnlock(Level level, String message, Optional lock) { - logger.at(level).log(message); - response.setPayload(message); - lock.ifPresent(Lock::release); - } - - /** Map over domains and refresh the DNS of those that reference the renamed hosts. */ - public static class RefreshDnsOnHostRenameMapper - extends Mapper { - - private static final long serialVersionUID = -5261698524424335531L; - private static final DnsQueue dnsQueue = DnsQueue.create(); - - private final ImmutableList refreshRequests; - private final Retrier retrier; - - RefreshDnsOnHostRenameMapper( - ImmutableList refreshRequests, Retrier retrier) { - this.refreshRequests = refreshRequests; - this.retrier = retrier; - } - - @Override - public final void map(@Nullable final DomainBase domain) { - if (domain == null) { - // Emit a single value so that the reducer always runs. The key and value don't matter. - emit(true, true); - return; - } - VKey referencingHostKey = null; - for (DnsRefreshRequest request : refreshRequests) { - if (isActive(domain, request.lastUpdateTime()) - && domain.getNameservers().contains(request.hostKey())) { - referencingHostKey = request.hostKey(); - break; - } - } - if (referencingHostKey != null) { - retrier.callWithRetry( - () -> dnsQueue.addDomainRefreshTask(domain.getDomainName()), - TransientFailureException.class); - logger.atInfo().log( - "Enqueued DNS refresh for domain %s referenced by host %s.", - domain.getDomainName(), referencingHostKey); - getContext().incrementCounter("domains refreshed"); - } else { - getContext().incrementCounter("domains not refreshed"); - } - - // Don't catch errors -- we allow the mapreduce to terminate on any errors that can't be - // resolved by retrying the transaction. The reducer only fires if the mapper completes - // without errors, meaning that it is acceptable to delete all tasks. - } - } - - /** - * A reducer that always fires exactly once. - * - *

This is really a reducer in name only; what it's really doing is waiting for all of the - * mapper tasks to finish, and then delete the pull queue tasks. Note that this only happens if - * the mapper completes execution without errors. - */ - public static class RefreshDnsOnHostRenameReducer extends Reducer { - - private static final long serialVersionUID = 9077366205249562118L; - - @NonFinalForTesting - private static AsyncTaskMetrics asyncTaskMetrics = new AsyncTaskMetrics(new SystemClock()); - - private final Lock lock; - private final Retrier retrier; - private final List refreshRequests; - - RefreshDnsOnHostRenameReducer( - List refreshRequests, Lock lock, Retrier retrier) { - this.refreshRequests = refreshRequests; - this.lock = lock; - this.retrier = retrier; - } - - @Override - public void reduce(Boolean key, ReducerInput values) { - // The reduce() method is run precisely once, because the NullInput caused the mapper to emit - // a dummy value once. - deleteTasksWithRetry( - refreshRequests, - getQueue(QUEUE_ASYNC_HOST_RENAME), - asyncTaskMetrics, - retrier, - OperationResult.SUCCESS); - - lock.release(); - } - } - - /** Deletes a list of tasks from the given queue using a retrier. */ - private static void deleteTasksWithRetry( - final List refreshRequests, - final Queue queue, - AsyncTaskMetrics asyncTaskMetrics, - Retrier retrier, - OperationResult result) { - if (refreshRequests.isEmpty()) { - return; - } - final List tasks = - refreshRequests.stream().map(DnsRefreshRequest::task).collect(toImmutableList()); - retrier.callWithRetry(() -> queue.deleteTask(tasks), TransientFailureException.class); - refreshRequests.forEach( - r -> asyncTaskMetrics.recordAsyncFlowResult(DNS_REFRESH, result, r.requestedTime())); - } - - /** A class that encapsulates the values of a request to refresh DNS for a renamed host. */ - @AutoValue - abstract static class DnsRefreshRequest implements Serializable { - - private static final long serialVersionUID = 1772812852271288622L; - - abstract VKey hostKey(); - - abstract DateTime lastUpdateTime(); - abstract DateTime requestedTime(); - abstract boolean isRefreshNeeded(); - abstract TaskHandle task(); - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setHostKey(VKey hostKey); - - abstract Builder setLastUpdateTime(DateTime lastUpdateTime); - abstract Builder setRequestedTime(DateTime requestedTime); - abstract Builder setIsRefreshNeeded(boolean isRefreshNeeded); - abstract Builder setTask(TaskHandle task); - abstract DnsRefreshRequest build(); - } - - /** - * Returns a packaged-up {@link DnsRefreshRequest} parsed from a task queue task. - */ - static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception { - ImmutableMap params = ImmutableMap.copyOf(task.extractParams()); - VKey hostKey = - VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified")); - HostResource host = - tm().transact(() -> tm().loadByKeyIfPresent(hostKey)) - .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist")); - boolean isHostDeleted = - isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp())); - if (isHostDeleted) { - logger.atInfo().log("Host %s is already deleted, not refreshing DNS.", hostKey); - } - return new AutoValue_RefreshDnsOnHostRenameAction_DnsRefreshRequest.Builder() - .setHostKey(hostKey) - .setLastUpdateTime(host.getUpdateTimestamp().getTimestamp()) - .setRequestedTime( - DateTime.parse( - checkNotNull(params.get(PARAM_REQUESTED_TIME), "Requested time not specified"))) - .setIsRefreshNeeded(!isHostDeleted) - .setTask(task) - .build(); - } - } -} diff --git a/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java b/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java index d170bf543..03b3501a0 100644 --- a/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java +++ b/core/src/main/java/google/registry/beam/common/RegistryJpaIO.java @@ -23,8 +23,6 @@ import com.google.common.collect.Streams; import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier; import google.registry.model.UpdateAutoTimestamp; import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.replay.SqlEntity; import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.persistence.transaction.TransactionManagerFactory; import java.io.Serializable; @@ -235,10 +233,6 @@ public final class RegistryJpaIO { @ProcessElement public void processElement(OutputReceiver outputReceiver) { - // Preload the migration schedule into cache, otherwise the cache loading query may happen - // before the setDatabaseSnapshot call in the transaction below, causing it to fail. - DatabaseMigrationStateSchedule.get(); - jpaTm() .transactNoRetry( () -> { @@ -433,11 +427,23 @@ public final class RegistryJpaIO { } } + /** Returns this entity's primary key field(s) in a string. */ private String toEntityKeyString(Object entity) { - if (entity instanceof SqlEntity) { - return ((SqlEntity) entity).getPrimaryKeyString(); + try { + return jpaTm() + .transact( + () -> + String.format( + "%s_%s", + entity.getClass().getSimpleName(), + jpaTm() + .getEntityManager() + .getEntityManagerFactory() + .getPersistenceUnitUtil() + .getIdentifier(entity))); + } catch (IllegalArgumentException e) { + return "Non-SqlEntity: " + entity; } - return "Non-SqlEntity: " + String.valueOf(entity); } } } diff --git a/core/src/main/java/google/registry/beam/initsql/BackupPaths.java b/core/src/main/java/google/registry/beam/initsql/BackupPaths.java deleted file mode 100644 index ea5b37f62..000000000 --- a/core/src/main/java/google/registry/beam/initsql/BackupPaths.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Strings.isNullOrEmpty; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Streams; -import google.registry.model.annotations.DeleteAfterMigration; -import org.joda.time.DateTime; - -/** - * Helpers for determining the fully qualified paths to Nomulus backup files. A backup consists of a - * Datastore export and Nomulus CommitLogs that overlap with the export. - */ -@DeleteAfterMigration -public final class BackupPaths { - - private BackupPaths() {} - - private static final String WILDCARD_CHAR = "*"; - private static final String EXPORT_PATTERN_TEMPLATE = "%s/all_namespaces/kind_%s/output-%s"; - - public static final String COMMIT_LOG_NAME_PREFIX = "commit_diff_until_"; - private static final String COMMIT_LOG_PATTERN_TEMPLATE = "%s/" + COMMIT_LOG_NAME_PREFIX + "*"; - - /** - * Pattern of the per-project file with Cloud SQL connection information. To get a concrete path, - * user needs to provide the name of the environment, alpha, crash, sandbox, or production. This - * file is meant for applications without access to secrets stored in Datastore. - * - *

In production, this is an base-64 encoded encrypted file with one line, which contains - * space-separated values of Cloud SQL instance name, login, and password. - * - *

A plain text may be used for tests to a local database. Replace Cloud SQL instance name with - * JDBC URL. - */ - private static final String SQL_CONN_INFO_FILE_PATTERN = - "gs://domain-registry-dev-deploy/cloudsql-credentials/%s/admin_credential.enc"; - - private static final ImmutableSet ALLOWED_ENV = - ImmutableSet.of("alpha", "crash", "sandbox", "production"); - - /** - * Returns a regex pattern that matches all Datastore export files of a given {@code kind}. - * - * @param exportDir path to the top directory of a Datastore export - * @param kind the 'kind' of the Datastore entity - */ - public static String getExportFileNamePattern(String exportDir, String kind) { - checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir."); - checkArgument(!isNullOrEmpty(kind), "Null or empty kind."); - return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, WILDCARD_CHAR); - } - - /** - * Returns an {@link ImmutableList} of regex patterns that match all Datastore export files of the - * given {@code kinds}. - * - * @param exportDir path to the top directory of a Datastore export - * @param kinds all entity 'kinds' to be matched - */ - public static ImmutableList getExportFilePatterns( - String exportDir, Iterable kinds) { - checkNotNull(kinds, "kinds"); - return Streams.stream(kinds) - .map(kind -> getExportFileNamePattern(exportDir, kind)) - .collect(ImmutableList.toImmutableList()); - } - - /** - * Returns the fully qualified path of a Datastore export file with the given {@code kind} and - * {@code shard}. - * - * @param exportDir path to the top directory of a Datastore export - * @param kind the 'kind' of the Datastore entity - * @param shard an integer suffix of the file name - */ - public static String getExportFileNameByShard(String exportDir, String kind, int shard) { - checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir."); - checkArgument(!isNullOrEmpty(kind), "Null or empty kind."); - checkArgument(shard >= 0, "Negative shard %s not allowed.", shard); - return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, shard); - } - - /** Returns an {@link ImmutableList} of regex patterns that match all CommitLog files. */ - public static ImmutableList getCommitLogFilePatterns(String commitLogDir) { - return ImmutableList.of(String.format(COMMIT_LOG_PATTERN_TEMPLATE, commitLogDir)); - } - - /** Gets the Commit timestamp from a CommitLog file name. */ - public static DateTime getCommitLogTimestamp(String fileName) { - checkArgument(!isNullOrEmpty(fileName), "Null or empty fileName."); - int start = fileName.lastIndexOf(COMMIT_LOG_NAME_PREFIX); - checkArgument(start >= 0, "Illegal file name %s.", fileName); - return DateTime.parse(fileName.substring(start + COMMIT_LOG_NAME_PREFIX.length())); - } - - public static ImmutableList getCloudSQLCredentialFilePatterns(String environmentName) { - checkArgument( - ALLOWED_ENV.contains(environmentName), "Invalid environment name %s", environmentName); - return ImmutableList.of(String.format(SQL_CONN_INFO_FILE_PATTERN, environmentName)); - } -} diff --git a/core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java b/core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java deleted file mode 100644 index a646a3d50..000000000 --- a/core/src/main/java/google/registry/beam/initsql/DomainBaseUtil.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.appengine.api.datastore.Entity; -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Objects; - -/** Helper for manipulating {@code DomainBase} when migrating from Datastore to SQL database */ -@DeleteAfterMigration -final class DomainBaseUtil { - - private DomainBaseUtil() {} - - /** - * Removes properties that contain foreign keys from a Datastore {@link Entity} that represents an - * Ofy {@link google.registry.model.domain.DomainBase}. This breaks the cycle of foreign key - * constraints between entity kinds, allowing {@code DomainBases} to be inserted into the SQL - * database. See {@link InitSqlPipeline} for a use case, where the full {@code DomainBases} are - * written again during the last stage of the pipeline. - * - *

The returned object may be in bad state. Specifically, {@link - * google.registry.model.eppcommon.StatusValue#INACTIVE} is not added after name servers are - * removed. This only impacts tests that manipulate Datastore entities directly. - * - *

This operation is performed on an Datastore {@link Entity} instead of Ofy Java object - * because Objectify requires access to a Datastore service when converting an Ofy object to a - * Datastore {@code Entity}. If we insist on working with Objectify objects, we face a few - * unsatisfactory options: - * - *

- * - *

Given our use case, operating on Datastore entities is the best option. - * - * @throws IllegalArgumentException if input does not represent a DomainBase - */ - static Entity removeBillingAndPollAndHosts(Entity domainBase) { - checkNotNull(domainBase, "domainBase"); - checkArgument( - Objects.equals(domainBase.getKind(), "DomainBase"), - "Expecting DomainBase, got %s", - domainBase.getKind()); - Entity clone = domainBase.clone(); - clone.removeProperty("autorenewBillingEvent"); - clone.removeProperty("autorenewPollMessage"); - clone.removeProperty("deletePollMessage"); - clone.removeProperty("nsHosts"); - domainBase.getProperties().keySet().stream() - .filter(s -> s.startsWith("transferData.")) - .forEach(s -> clone.removeProperty(s)); - domainBase.getProperties().keySet().stream() - .filter(s -> s.startsWith("gracePeriods.")) - .forEach(s -> clone.removeProperty(s)); - return clone; - } -} diff --git a/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java b/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java deleted file mode 100644 index 3f17dcc3e..000000000 --- a/core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.backup.VersionedEntity; -import google.registry.beam.common.RegistryJpaIO; -import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.billing.BillingEvent; -import google.registry.model.common.Cursor; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.token.AllocationToken; -import google.registry.model.host.HostResource; -import google.registry.model.poll.PollMessage; -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.tld.Registry; -import google.registry.persistence.PersistenceModule.TransactionIsolationLevel; -import java.io.Serializable; -import java.util.Collection; -import java.util.Optional; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.transforms.Wait; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.joda.time.DateTime; - -/** - * A BEAM pipeline that populates a SQL database with data from a Datastore backup. - * - *

This pipeline migrates EPP resources and related entities that cross-reference each other. To - * avoid violating foreign key constraints, writes to SQL are ordered by entity kinds. In addition, - * the {@link DomainBase} kind is written twice (see details below). The write order is presented - * below. Although some kinds can be written concurrently, e.g. {@code ContactResource} and {@code - * RegistrarContact}, we do not expect any performance benefit since the limiting resource is the - * number of JDBC connections. Google internal users may refer to the design doc for more information. - * - *

    - *
  1. {@link Registry}: Assumes that {@code PremiumList} and {@code ReservedList} have been set - * up in the SQL database. - *
  2. {@link Cursor}: Logically can depend on {@code Registry}, but without foreign key. - *
  3. {@link Registrar}: Logically depends on {@code Registry}, Foreign key not modeled yet. - *
  4. {@link ContactResource}: references {@code Registrar} - *
  5. {@link RegistrarContact}: references {@code Registrar}. - *
  6. Cleansed {@link DomainBase}: with references to {@code BillingEvent}, {@code Recurring}, - * {@code Cancellation} and {@code HostResource} removed, still references {@code Registrar} - * and {@code ContactResource}. The removal breaks circular Foreign Key references. - *
  7. {@link HostResource}: references {@code DomainBase}. - *
  8. {@link HistoryEntry}: maps to one of three SQL entity types and may reference {@code - * Registrar}, {@code ContactResource}, {@code HostResource}, and {@code DomainBase}. - *
  9. {@link AllocationToken}: references {@code HistoryEntry}. - *
  10. {@link BillingEvent.Recurring}: references {@code Registrar}, {@code DomainBase} and {@code - * HistoryEntry}. - *
  11. {@link BillingEvent.OneTime}: references {@code Registrar}, {@code DomainBase}, {@code - * BillingEvent.Recurring}, {@code HistoryEntry} and {@code AllocationToken}. - *
  12. {@link BillingEvent.Cancellation}: references {@code Registrar}, {@code DomainBase}, {@code - * BillingEvent.Recurring}, {@code BillingEvent.OneTime}, and {@code HistoryEntry}. - *
  13. {@link PollMessage}: references {@code Registrar}, {@code DomainBase}, {@code - * ContactResource}, {@code HostResource}, and {@code HistoryEntry}. - *
  14. {@link DomainBase}, original copy from Datastore. - *
- * - *

This pipeline expects that the source Datastore has at least one entity in each of the types - * above. This assumption allows us to construct a simpler pipeline graph that can be visually - * examined, and is true in all intended use cases. However, tests must not violate this assumption - * when setting up data, otherwise they may run into foreign key constraint violations. The reason - * is that this pipeline uses the {@link Wait} transform to order the persistence by entity type. - * However, the wait is skipped if the target type has no data, resulting in subsequent entity types - * starting prematurely. E.g., if a Datastore has no {@code RegistrarContact} entities, the pipeline - * may start writing {@code DomainBase} entities before all {@code Registry}, {@code Registrar} and - * {@code ContactResource} entities have been persisted. - */ -@DeleteAfterMigration -public class InitSqlPipeline implements Serializable { - - /** - * Datastore kinds to be written to the SQL database before the cleansed version of {@link - * DomainBase}. - */ - private static final ImmutableList> PHASE_ONE_ORDERED = - ImmutableList.of( - Registry.class, - Cursor.class, - Registrar.class, - ContactResource.class, - RegistrarContact.class); - - /** - * Datastore kinds to be written to the SQL database after the cleansed version of {@link - * DomainBase}. - */ - private static final ImmutableList> PHASE_TWO_ORDERED = - ImmutableList.of( - HostResource.class, - HistoryEntry.class, - AllocationToken.class, - BillingEvent.Recurring.class, - BillingEvent.OneTime.class, - BillingEvent.Cancellation.class, - PollMessage.class, - DomainBase.class); - - private final InitSqlPipelineOptions options; - - InitSqlPipeline(InitSqlPipelineOptions options) { - this.options = options; - } - - PipelineResult run() { - return run(Pipeline.create(options)); - } - - @VisibleForTesting - PipelineResult run(Pipeline pipeline) { - setupPipeline(pipeline); - return pipeline.run(); - } - - @VisibleForTesting - void setupPipeline(Pipeline pipeline) { - options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_UNCOMMITTED); - PCollectionTuple datastoreSnapshot = - pipeline.apply( - "Load Datastore snapshot", - Transforms.loadDatastoreSnapshot( - options.getDatastoreExportDir(), - options.getCommitLogDir(), - DateTime.parse(options.getCommitLogStartTimestamp()), - DateTime.parse(options.getCommitLogEndTimestamp()), - ImmutableSet.builder() - .add("DomainBase") - .addAll(toKindStrings(PHASE_ONE_ORDERED)) - .addAll(toKindStrings(PHASE_TWO_ORDERED)) - .build())); - - // Set up the pipeline to write entity kinds from PHASE_ONE_ORDERED to SQL. Return an object - // that signals the completion of the phase. - PCollection blocker = - scheduleOnePhaseWrites(datastoreSnapshot, PHASE_ONE_ORDERED, Optional.empty(), null); - blocker = - writeToSql( - "DomainBase without circular foreign keys", - removeDomainBaseForeignKeys(datastoreSnapshot) - .apply("Wait on phase one", Wait.on(blocker))); - // Set up the pipeline to write entity kinds from PHASE_TWO_ORDERED to SQL. This phase won't - // start until all cleansed DomainBases have been written (started by line above). - scheduleOnePhaseWrites( - datastoreSnapshot, PHASE_TWO_ORDERED, Optional.of(blocker), "DomainBaseNoFkeys"); - } - - private PCollection removeDomainBaseForeignKeys( - PCollectionTuple datastoreSnapshot) { - PCollection domainBases = - datastoreSnapshot.get(Transforms.createTagForKind("DomainBase")); - return domainBases.apply( - "Remove circular foreign keys from DomainBase", - ParDo.of(new RemoveDomainBaseForeignKeys())); - } - - /** - * Sets up the pipeline to write entities in {@code entityClasses} to SQL. Entities are written - * one kind at a time based on each kind's position in {@code entityClasses}. Concurrency exists - * within each kind. - * - * @param datastoreSnapshot the Datastore snapshot of all data to be migrated to SQL - * @param entityClasses the entity types in write order - * @param blockingPCollection the pipeline stage that blocks this phase - * @param blockingTag description of the stage (if exists) that blocks this phase. Needed for - * generating unique transform ids - * @return the output {@code PCollection} from the writing of the last entity kind. Other parts of - * the pipeline can {@link Wait} on this object - */ - private PCollection scheduleOnePhaseWrites( - PCollectionTuple datastoreSnapshot, - Collection> entityClasses, - Optional> blockingPCollection, - String blockingTag) { - checkArgument(!entityClasses.isEmpty(), "Each phase must have at least one kind."); - ImmutableList> tags = - toKindStrings(entityClasses).stream() - .map(Transforms::createTagForKind) - .collect(ImmutableList.toImmutableList()); - - PCollection prev = blockingPCollection.orElse(null); - String prevTag = blockingTag; - for (TupleTag tag : tags) { - PCollection curr = datastoreSnapshot.get(tag); - if (prev != null) { - curr = curr.apply("Wait on " + prevTag, Wait.on(prev)); - } - prev = writeToSql(tag.getId(), curr); - prevTag = tag.getId(); - } - return prev; - } - - private PCollection writeToSql(String transformId, PCollection data) { - return data.apply( - "Write to Sql: " + transformId, - RegistryJpaIO.write() - .withName(transformId) - .withBatchSize(options.getSqlWriteBatchSize()) - .withShards(options.getSqlWriteShards()) - .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity) - .disableUpdateAutoTimestamp()); - } - - private static ImmutableList toKindStrings(Collection> entityClasses) { - return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList()); - } - - public static void main(String[] args) { - InitSqlPipelineOptions options = - PipelineOptionsFactory.fromArgs(args).withValidation().as(InitSqlPipelineOptions.class); - new InitSqlPipeline(options).run(); - } -} diff --git a/core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java b/core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java deleted file mode 100644 index b4f3a1940..000000000 --- a/core/src/main/java/google/registry/beam/initsql/InitSqlPipelineOptions.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import google.registry.beam.common.RegistryPipelineOptions; -import google.registry.model.annotations.DeleteAfterMigration; -import org.apache.beam.sdk.options.Description; -import org.apache.beam.sdk.options.Validation; - -/** Pipeline options for {@link InitSqlPipeline} */ -@DeleteAfterMigration -public interface InitSqlPipelineOptions extends RegistryPipelineOptions { - - @Description("The root directory of the export to load.") - String getDatastoreExportDir(); - - void setDatastoreExportDir(String datastoreExportDir); - - @Description("The directory that contains all CommitLog files.") - String getCommitLogDir(); - - void setCommitLogDir(String commitLogDir); - - @Description("The earliest CommitLogs to load, in ISO8601 format.") - @Validation.Required - String getCommitLogStartTimestamp(); - - void setCommitLogStartTimestamp(String commitLogStartTimestamp); - - @Description("The latest CommitLogs to load, in ISO8601 format.") - @Validation.Required - String getCommitLogEndTimestamp(); - - void setCommitLogEndTimestamp(String commitLogEndTimestamp); -} diff --git a/core/src/main/java/google/registry/beam/initsql/README.md b/core/src/main/java/google/registry/beam/initsql/README.md deleted file mode 100644 index 19c54200e..000000000 --- a/core/src/main/java/google/registry/beam/initsql/README.md +++ /dev/null @@ -1,17 +0,0 @@ -## Summary - -This package contains a BEAM pipeline that populates a Cloud SQL database from a -Datastore backup. The pipeline uses an unsynchronized Datastore export and -overlapping CommitLogs generated by the Nomulus server to recreate a consistent -Datastore snapshot, and writes the data to a Cloud SQL instance. - -## Pipeline Visualization - -The golden flow graph of the InitSqlPipeline is saved both as a text-base -[DOT file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.dot) -and a -[.png file](../../../../../../test/resources/google/registry/beam/initsql/pipeline_golden.png). -A test compares the flow graph of the current pipeline with the golden graph, -and will fail if changes are detected. When this happens, run the Gradle task -':core:updateInitSqlPipelineGraph' to update the golden files and review the -changes. diff --git a/core/src/main/java/google/registry/beam/initsql/Transforms.java b/core/src/main/java/google/registry/beam/initsql/Transforms.java deleted file mode 100644 index fa969592b..000000000 --- a/core/src/main/java/google/registry/beam/initsql/Transforms.java +++ /dev/null @@ -1,485 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; -import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp; -import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static google.registry.util.DateTimeUtils.isBeforeOrAt; -import static google.registry.util.DomainNameUtils.canonicalizeHostname; -import static java.util.Comparator.comparing; -import static org.apache.beam.sdk.values.TypeDescriptors.kvs; -import static org.apache.beam.sdk.values.TypeDescriptors.strings; - -import avro.shaded.com.google.common.collect.Iterators; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityTranslator; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Streams; -import google.registry.backup.CommitLogImports; -import google.registry.backup.VersionedEntity; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.billing.BillingEvent.Flag; -import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.domain.DomainBase; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.SqlEntity; -import google.registry.model.reporting.HistoryEntry; -import google.registry.tools.LevelDbLogReader; -import java.io.Serializable; -import java.util.Collection; -import java.util.Iterator; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import javax.annotation.Nullable; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.Compression; -import org.apache.beam.sdk.io.FileIO; -import org.apache.beam.sdk.io.FileIO.ReadableFile; -import org.apache.beam.sdk.io.fs.EmptyMatchTreatment; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.Flatten; -import org.apache.beam.sdk.transforms.GroupByKey; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.transforms.ProcessFunction; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionList; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.sdk.values.TupleTagList; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.joda.time.DateTime; - -/** - * {@link PTransform Pipeline transforms} used in pipelines that load from both Datastore export - * files and Nomulus CommitLog files. - */ -@DeleteAfterMigration -public final class Transforms { - - private Transforms() {} - - /** - * The commitTimestamp assigned to all entities loaded from a Datastore export file. The exact - * value does not matter, but it must be lower than the timestamps of real CommitLog records. - */ - @VisibleForTesting static final long EXPORT_ENTITY_TIME_STAMP = START_OF_TIME.getMillis(); - - /** - * Returns a {@link TupleTag} that can be used to retrieve entities of the given {@code kind} from - * the Datastore snapshot returned by {@link #loadDatastoreSnapshot}. - */ - public static TupleTag createTagForKind(String kind) { - // When used with PCollectionTuple the result must retain generic type information. - // Both the Generic param and the empty bracket below are important. - return new TupleTag(Transforms.class.getSimpleName() + ":" + kind) {}; - } - - /** - * Composite {@link PTransform transform} that loads the Datastore snapshot right before {@code - * commitLogToTime} for caller specified {@code kinds}. The resulting snapshot has all changes - * that happened before {@code commitLogToTime}, and none at or after {@code commitLogToTime}. - * - *

Caller must provide the location of a Datastore export that started AFTER {@code - * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory - * of all CommitLog files. - * - *

Selection of {@code commitLogFromTime} and {@code commitLogToTime} should follow the - * guidelines below to ensure that all incremental changes concurrent with the export are covered: - * - *

- * - *

The output from the returned transform is a {@link PCollectionTuple} consisting of {@link - * VersionedEntity VersionedEntities} grouped into {@link PCollection PCollections} by {@code - * kind}. - */ - public static PTransform loadDatastoreSnapshot( - String exportDir, - String commitLogDir, - DateTime commitLogFromTime, - DateTime commitLogToTime, - Set kinds) { - checkArgument(kinds != null && !kinds.isEmpty(), "At least one kind is expected."); - - // Create tags to collect entities by kind in final step. - final ImmutableMap> outputTags = - kinds.stream() - .collect(ImmutableMap.toImmutableMap(kind -> kind, Transforms::createTagForKind)); - // Arbitrarily select one tag as mainOutTag and put the remaining ones in a TupleTagList. - // This separation is required by ParDo's config API. - Iterator> tagsIt = outputTags.values().iterator(); - final TupleTag mainOutputTag = tagsIt.next(); - final TupleTagList additionalTags = TupleTagList.of(ImmutableList.copyOf(tagsIt)); - - return new PTransform() { - @Override - public PCollectionTuple expand(PBegin input) { - PCollection exportedEntities = - input - .apply("Get export file patterns", getDatastoreExportFilePatterns(exportDir, kinds)) - .apply("Find export files", getFilesByPatterns()) - .apply("Load export data", loadExportDataFromFiles()); - PCollection commitLogEntities = - input - .apply("Get commitlog file patterns", getCommitLogFilePatterns(commitLogDir)) - .apply("Find commitlog files", getFilesByPatterns()) - .apply( - "Filter commitLog by time", - filterCommitLogsByTime(commitLogFromTime, commitLogToTime)) - .apply("Load commitlog data", loadCommitLogsFromFiles(kinds)); - return PCollectionList.of(exportedEntities) - .and(commitLogEntities) - .apply("Merge exports and CommitLogs", Flatten.pCollections()) - .apply( - "Key entities by Datastore Keys", - // Converting to KV instead of KV b/c default coder for Key - // (SerializableCoder) is not deterministic and cannot be used with GroupBy. - MapElements.into(kvs(strings(), TypeDescriptor.of(VersionedEntity.class))) - .via((VersionedEntity e) -> KV.of(e.key().toString(), e))) - .apply("Gather entities by key", GroupByKey.create()) - .apply( - "Output latest version per entity", - ParDo.of( - new DoFn>, VersionedEntity>() { - @ProcessElement - public void processElement( - @Element KV> kv, - MultiOutputReceiver out) { - Optional latest = - Streams.stream(kv.getValue()) - .sorted(comparing(VersionedEntity::commitTimeMills).reversed()) - .findFirst(); - // Throw to abort (after default retries). Investigate, fix, and rerun. - checkState( - latest.isPresent(), "Unexpected key with no data", kv.getKey()); - if (latest.get().isDelete()) { - return; - } - String kind = latest.get().getEntity().get().getKind(); - out.get(outputTags.get(kind)).output(latest.get()); - } - }) - .withOutputTags(mainOutputTag, additionalTags)); - } - }; - } - - /** - * Returns a {@link PTransform transform} that can generate a collection of patterns that match - * all Datastore CommitLog files. - */ - public static PTransform> getCommitLogFilePatterns( - String commitLogDir) { - return toStringPCollection(BackupPaths.getCommitLogFilePatterns(commitLogDir)); - } - - /** - * Returns a {@link PTransform transform} that can generate a collection of patterns that match - * all Datastore export files of the given {@code kinds}. - */ - public static PTransform> getDatastoreExportFilePatterns( - String exportDir, Collection kinds) { - return toStringPCollection(getExportFilePatterns(exportDir, kinds)); - } - - public static PTransform> getCloudSqlConnectionInfoFilePatterns( - String gcpProjectName) { - return toStringPCollection(BackupPaths.getCloudSQLCredentialFilePatterns(gcpProjectName)); - } - - /** - * Returns a {@link PTransform} from file name patterns to file {@link Metadata Metadata records}. - */ - public static PTransform, PCollection> getFilesByPatterns() { - return new PTransform, PCollection>() { - @Override - public PCollection expand(PCollection input) { - return input.apply(FileIO.matchAll().withEmptyMatchTreatment(EmptyMatchTreatment.ALLOW)); - } - }; - } - - /** - * Returns CommitLog files with timestamps between {@code fromTime} (inclusive) and {@code - * endTime} (exclusive). - */ - public static PTransform, PCollection> - filterCommitLogsByTime(DateTime fromTime, DateTime toTime) { - return ParDo.of(new FilterCommitLogFileByTime(fromTime, toTime)); - } - - /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */ - public static PTransform, PCollection> - loadExportDataFromFiles() { - return processFiles( - new BackupFileReader( - file -> - Iterators.transform( - LevelDbLogReader.from(file.open()), - (byte[] bytes) -> VersionedEntity.from(EXPORT_ENTITY_TIME_STAMP, bytes)))); - } - - /** Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity}. */ - public static PTransform, PCollection> - loadCommitLogsFromFiles(Set kinds) { - return processFiles( - new BackupFileReader( - file -> - CommitLogImports.loadEntities(file.open()).stream() - .filter(e -> kinds.contains(e.key().getKind())) - .iterator())); - } - - // Production data repair configs go below. See b/185954992. Note that the CommitLog replay - // process does not filter out the ignored entities listed below, a mistake that we do not fix - // for operational convenience. Instead, the Database comparison tool will filter them out. See - // ValidateSqlUtils.java for more information. - - // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized - // history. They can be safely ignored. - public static final ImmutableSet IGNORED_DOMAINS = - ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT"); - - // Prober hosts referencing phantom registrars. They and their associated history entries can be - // safely ignored. - public static final ImmutableSet IGNORED_HOSTS = - ImmutableSet.of( - "4E21_WJ0TEST-GOOGLE", - "4E21_WJ1TEST-GOOGLE", - "4E21_WJ2TEST-GOOGLE", - "4E21_WJ3TEST-GOOGLE"); - - // Prober contacts referencing phantom registrars. They and their associated history entries can - // be safely ignored. - public static final ImmutableSet IGNORED_CONTACTS = - ImmutableSet.of( - "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE"); - - private static boolean isMigratable(Entity entity) { - // Checks specific to production data. See b/185954992 for details. - // The names of these bad entities in production do not conflict with other environments. For - // simplicities sake we apply them regardless of the source of the data. - if (entity.getKind().equals("DomainBase") - && IGNORED_DOMAINS.contains(entity.getKey().getName())) { - return false; - } - if (entity.getKind().equals("ContactResource")) { - String roid = entity.getKey().getName(); - return !IGNORED_CONTACTS.contains(roid); - } - if (entity.getKind().equals("HostResource")) { - String roid = entity.getKey().getName(); - return !IGNORED_HOSTS.contains(roid); - } - if (entity.getKind().equals("HistoryEntry")) { - // DOMAIN_APPLICATION_CREATE is deprecated type and should not be migrated. - // The Enum name DOMAIN_APPLICATION_CREATE no longer exists in Java and cannot - // be deserialized. - if (Objects.equals(entity.getProperty("type"), "DOMAIN_APPLICATION_CREATE")) { - return false; - } - - // Remove production bad data: Histories of ignored EPP resources: - com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent(); - if (parentKey.getKind().equals("ContactResource")) { - String contactRoid = parentKey.getName(); - return !IGNORED_CONTACTS.contains(contactRoid); - } - if (parentKey.getKind().equals("HostResource")) { - String hostRoid = parentKey.getName(); - return !IGNORED_HOSTS.contains(hostRoid); - } - if (parentKey.getKind().equals("DomainBase")) { - String domainRoid = parentKey.getName(); - return !IGNORED_DOMAINS.contains(domainRoid); - } - } - return true; - } - - @VisibleForTesting - static Entity repairBadData(Entity entity) { - if (entity.getKind().equals("Cancellation") - && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) { - // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the - // AUTO_RENEW flag. Note: all affected entities have empty flags so we can simply assign - // instead of append. See b/185954992. - entity.setUnindexedProperty("reason", Reason.RENEW.name()); - entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name())); - } else if (entity.getKind().equals("DomainBase")) { - // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this. - entity.setIndexedProperty( - "fullyQualifiedDomainName", - canonicalizeHostname((String) entity.getProperty("fullyQualifiedDomainName"))); - } else if (entity.getKind().equals("HostResource")) { - entity.setIndexedProperty( - "fullyQualifiedHostName", - canonicalizeHostname((String) entity.getProperty("fullyQualifiedHostName"))); - } - return entity; - } - - private static SqlEntity toSqlEntity(Object ofyEntity) { - if (ofyEntity instanceof HistoryEntry) { - HistoryEntry ofyHistory = (HistoryEntry) ofyEntity; - return (SqlEntity) ofyHistory.toChildHistoryEntity(); - } - return ((DatastoreAndSqlEntity) ofyEntity).toSqlEntity().get(); - } - - /** - * Converts a {@link VersionedEntity} to an JPA entity for persistence. - * - * @return An object to be persisted to SQL, or null if the input is not to be migrated. (Not - * using Optional in return because as a one-use method, we do not want to invest the effort - * to make Optional work with BEAM) - */ - @Nullable - public static SqlEntity convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) { - return dsEntity - .getEntity() - .filter(Transforms::isMigratable) - .map(Transforms::repairBadData) - .map(e -> auditedOfy().toPojo(e)) - .map(Transforms::toSqlEntity) - .orElse(null); - } - - /** Interface for serializable {@link Supplier suppliers}. */ - public interface SerializableSupplier extends Supplier, Serializable {} - - /** - * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the - * given {@link Iterable}. - */ - private static PTransform> toStringPCollection( - Iterable strings) { - return Create.of(strings).withCoder(StringUtf8Coder.of()); - } - - /** - * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using - * caller-provided {@code transformer}. - */ - private static PTransform, PCollection> processFiles( - DoFn transformer) { - return new PTransform, PCollection>() { - @Override - public PCollection expand(PCollection input) { - return input - .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED)) - .apply(transformer.getClass().getSimpleName(), ParDo.of(transformer)); - } - }; - } - - private static class FilterCommitLogFileByTime extends DoFn { - private final DateTime fromTime; - private final DateTime toTime; - - FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) { - checkNotNull(fromTime, "fromTime"); - checkNotNull(toTime, "toTime"); - checkArgument( - fromTime.isBefore(toTime), - "Invalid time range: fromTime (%s) is before endTime (%s)", - fromTime, - toTime); - this.fromTime = fromTime; - this.toTime = toTime; - } - - @ProcessElement - public void processElement(@Element Metadata fileMeta, OutputReceiver out) { - DateTime timestamp = getCommitLogTimestamp(fileMeta.resourceId().toString()); - if (isBeforeOrAt(fromTime, timestamp) && timestamp.isBefore(toTime)) { - out.output(fileMeta); - } - } - } - - /** - * Reads from a Datastore backup file and converts its content into {@link VersionedEntity - * VersionedEntities}. - * - *

The input file may be either a LevelDb file from a Datastore export or a CommitLog file - * generated by the Nomulus server. In either case, the file contains variable-length records and - * must be read sequentially from the beginning. If the read fails, the file needs to be retried - * from the beginning. - */ - private static class BackupFileReader extends DoFn { - private final ProcessFunction> reader; - - private BackupFileReader(ProcessFunction> reader) { - this.reader = reader; - } - - @ProcessElement - public void processElement(@Element ReadableFile file, OutputReceiver out) { - try { - reader.apply(file).forEachRemaining(out::output); - } catch (Exception e) { - // Let the pipeline use default retry strategy on the whole file. For GCP Dataflow this - // means retrying up to 4 times (may include other files grouped with this one), and failing - // the pipeline if no success. - throw new RuntimeException(e); - } - } - } - - /** - * Removes BillingEvents, {@link google.registry.model.poll.PollMessage PollMessages} and {@link - * google.registry.model.host.HostResource} from a {@link DomainBase}. These are circular foreign - * key constraints that prevent migration of {@code DomainBase} to SQL databases. - * - *

See {@link InitSqlPipeline} for more information. - */ - static class RemoveDomainBaseForeignKeys extends DoFn { - - @ProcessElement - public void processElement( - @Element VersionedEntity domainBase, OutputReceiver out) { - checkArgument( - domainBase.getEntity().isPresent(), "Unexpected delete entity %s", domainBase.key()); - Entity outputEntity = - DomainBaseUtil.removeBillingAndPollAndHosts(domainBase.getEntity().get()); - out.output( - VersionedEntity.from( - domainBase.commitTimeMills(), - EntityTranslator.convertToPb(outputEntity).toByteArray())); - } - } -} diff --git a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java index 44ad0a341..bb368b547 100644 --- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java +++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java @@ -41,11 +41,9 @@ import java.util.Optional; import java.util.regex.Pattern; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.coders.SerializableCoder; import org.apache.beam.sdk.coders.StringUtf8Coder; import org.apache.beam.sdk.io.FileIO; import org.apache.beam.sdk.io.TextIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.Contextful; import org.apache.beam.sdk.transforms.Count; @@ -92,28 +90,11 @@ public class InvoicingPipeline implements Serializable { void setupPipeline(Pipeline pipeline) { options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED); - PCollection billingEvents = - options.getDatabase().equals("DATASTORE") - ? readFromBigQuery(options, pipeline) - : readFromCloudSql(options, pipeline); - + PCollection billingEvents = readFromCloudSql(options, pipeline); saveInvoiceCsv(billingEvents, options); - saveDetailedCsv(billingEvents, options); } - static PCollection readFromBigQuery( - InvoicingPipelineOptions options, Pipeline pipeline) { - return pipeline.apply( - "Read BillingEvents from Bigquery", - BigQueryIO.read(BillingEvent::parseFromRecord) - .fromQuery(makeQuery(options.getYearMonth(), options.getProject())) - .withCoder(SerializableCoder.of(BillingEvent.class)) - .usingStandardSql() - .withoutValidation() - .withTemplateCompatibility()); - } - static PCollection readFromCloudSql( InvoicingPipelineOptions options, Pipeline pipeline) { Read read = diff --git a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java index 9a9fab709..ff4d5a69d 100644 --- a/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java +++ b/core/src/main/java/google/registry/beam/invoicing/InvoicingPipelineOptions.java @@ -30,11 +30,6 @@ public interface InvoicingPipelineOptions extends RegistryPipelineOptions { void setInvoiceFilePrefix(String value); - @Description("The database to read data from.") - String getDatabase(); - - void setDatabase(String value); - @Description("The GCS bucket URL for invoices and detailed reports to be uploaded.") String getBillingBucketUrl(); diff --git a/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java b/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java index 4107ea539..cdfa69e35 100644 --- a/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java +++ b/core/src/main/java/google/registry/beam/spec11/DomainNameInfo.java @@ -14,30 +14,19 @@ package google.registry.beam.spec11; -import static google.registry.beam.BeamUtils.checkFieldsNotNull; -import static google.registry.beam.BeamUtils.extractField; - import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; import java.io.Serializable; -import org.apache.avro.generic.GenericRecord; -import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; /** * A POJO representing a domain name and associated info, parsed from a {@code SchemaAndRecord}. * - *

This is a trivially serializable class that allows Beam to transform the results of a Bigquery - * query into a standard Java representation, giving us the type guarantees and ease of manipulation - * Bigquery lacks, while localizing any Bigquery-side failures to the {@link #parseFromRecord} - * function. + *

This is a trivially serializable class that allows Beam to transform the results of a SQL + * query into a standard Java representation. */ @AutoValue public abstract class DomainNameInfo implements Serializable { - private static final ImmutableList FIELD_NAMES = - ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress"); - /** Returns the fully qualified domain name. */ abstract String domainName(); @@ -50,28 +39,8 @@ public abstract class DomainNameInfo implements Serializable { /** Returns the email address of the registrar associated with this domain. */ abstract String registrarEmailAddress(); - /** - * Constructs a {@link DomainNameInfo} from an Apache Avro {@code SchemaAndRecord}. - * - * @see - * Apache AVRO GenericRecord - */ - static DomainNameInfo parseFromRecord(SchemaAndRecord schemaAndRecord) { - checkFieldsNotNull(FIELD_NAMES, schemaAndRecord); - GenericRecord record = schemaAndRecord.getRecord(); - return create( - extractField(record, "domainName"), - extractField(record, "domainRepoId"), - extractField(record, "registrarId"), - extractField(record, "registrarEmailAddress")); - } - /** * Creates a concrete {@link DomainNameInfo}. - * - *

This should only be used outside this class for testing- instances of {@link DomainNameInfo} - * should otherwise come from {@link #parseFromRecord}. */ @VisibleForTesting static DomainNameInfo create( diff --git a/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java b/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java index c11b55cfe..437de19ed 100644 --- a/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java +++ b/core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java @@ -15,7 +15,6 @@ package google.registry.beam.spec11; import static com.google.common.base.Preconditions.checkArgument; -import static google.registry.beam.BeamUtils.getQueryFromFile; import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import com.google.auto.value.AutoValue; @@ -33,15 +32,12 @@ import google.registry.model.reporting.Spec11ThreatMatch.ThreatType; import google.registry.persistence.PersistenceModule.TransactionIsolationLevel; import google.registry.persistence.VKey; import google.registry.util.Retrier; -import google.registry.util.SqlTemplate; import google.registry.util.UtilsModule; import java.io.Serializable; import javax.inject.Singleton; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.coders.SerializableCoder; import org.apache.beam.sdk.io.TextIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.GroupByKey; @@ -104,10 +100,7 @@ public class Spec11Pipeline implements Serializable { void setupPipeline(Pipeline pipeline) { options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED); - PCollection domains = - options.getDatabase().equals("DATASTORE") - ? readFromBigQuery(options, pipeline) - : readFromCloudSql(pipeline); + PCollection domains = readFromCloudSql(pipeline); PCollection> threatMatches = domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn)); @@ -156,24 +149,6 @@ public class Spec11Pipeline implements Serializable { })); } - static PCollection readFromBigQuery( - Spec11PipelineOptions options, Pipeline pipeline) { - return pipeline.apply( - "Read active domains from BigQuery", - BigQueryIO.read(DomainNameInfo::parseFromRecord) - .fromQuery( - SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "domain_name_infos.sql")) - .put("PROJECT_ID", options.getProject()) - .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export") - .put("REGISTRAR_TABLE", "Registrar") - .put("DOMAIN_BASE_TABLE", "DomainBase") - .build()) - .withCoder(SerializableCoder.of(DomainNameInfo.class)) - .usingStandardSql() - .withoutValidation() - .withTemplateCompatibility()); - } - private static KV parseRow(Object[] row) { return KV.of((String) row[0], (String) row[1]); } diff --git a/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java b/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java index a04730b7c..7e3ab546b 100644 --- a/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java +++ b/core/src/main/java/google/registry/beam/spec11/Spec11PipelineOptions.java @@ -34,9 +34,4 @@ public interface Spec11PipelineOptions extends RegistryPipelineOptions { String getReportingBucketUrl(); void setReportingBucketUrl(String value); - - @Description("The database to read data from.") - String getDatabase(); - - void setDatabase(String value); } diff --git a/core/src/main/java/google/registry/flows/EppException.java b/core/src/main/java/google/registry/flows/EppException.java index c4dd987f8..5dbef1faa 100644 --- a/core/src/main/java/google/registry/flows/EppException.java +++ b/core/src/main/java/google/registry/flows/EppException.java @@ -25,7 +25,6 @@ import google.registry.model.eppinput.EppInput.InnerCommand; import google.registry.model.eppinput.EppInput.ResourceCommandWrapper; import google.registry.model.eppoutput.Result; import google.registry.model.eppoutput.Result.Code; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import java.lang.annotation.Documented; import java.lang.annotation.Inherited; import java.lang.annotation.Retention; @@ -262,12 +261,4 @@ public abstract class EppException extends Exception { super("Specified protocol version is not implemented"); } } - - /** Registry is currently undergoing maintenance and is in read-only mode. */ - @EppResultCode(Code.COMMAND_FAILED) - public static class ReadOnlyModeEppException extends EppException { - ReadOnlyModeEppException(ReadOnlyModeException cause) { - super("Registry is currently undergoing maintenance and is in read-only mode", cause); - } - } } diff --git a/core/src/main/java/google/registry/flows/FlowRunner.java b/core/src/main/java/google/registry/flows/FlowRunner.java index d7747fc98..e1859ba7c 100644 --- a/core/src/main/java/google/registry/flows/FlowRunner.java +++ b/core/src/main/java/google/registry/flows/FlowRunner.java @@ -19,7 +19,6 @@ import static google.registry.xml.XmlTransformer.prettyPrint; import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; -import google.registry.flows.EppException.ReadOnlyModeEppException; import google.registry.flows.FlowModule.DryRun; import google.registry.flows.FlowModule.InputXml; import google.registry.flows.FlowModule.RegistrarId; @@ -29,7 +28,6 @@ import google.registry.flows.session.LoginFlow; import google.registry.model.eppcommon.Trid; import google.registry.model.eppoutput.EppOutput; import google.registry.monitoring.whitebox.EppMetric; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import javax.inject.Inject; import javax.inject.Provider; @@ -99,8 +97,6 @@ public class FlowRunner { return e.output; } catch (EppRuntimeException e) { throw e.getCause(); - } catch (ReadOnlyModeException e) { - throw new ReadOnlyModeEppException(e); } } diff --git a/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java b/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java index 8a6faf9a1..00c1230dd 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java @@ -50,7 +50,6 @@ import org.joda.time.DateTime; /** * An EPP flow that creates a new contact. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link ResourceAlreadyExistsForThisClientException} * @error {@link ResourceCreateContentionException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java b/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java index f8f636277..6e5a53627 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java @@ -24,7 +24,6 @@ import static google.registry.model.ResourceTransferUtils.denyPendingTransfer; import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete; import static google.registry.model.eppoutput.Result.Code.SUCCESS; import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import com.google.common.collect.ImmutableSet; @@ -58,7 +57,6 @@ import org.joda.time.DateTime; * references to the host before the deletion is allowed to proceed. A poll message will be written * with the success or failure message when the process is complete. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException} @@ -92,7 +90,6 @@ public final class ContactDeleteFlow implements TransactionalFlow { extensionManager.register(MetadataExtension.class); validateRegistrarIsLoggedIn(registrarId); extensionManager.validate(); - assertAsyncActionsAreAllowed(); DateTime now = tm().getTransactionTime(); checkLinkedDomains(targetId, now, ContactResource.class); ContactResource existingContact = loadAndVerifyExistence(ContactResource.class, targetId, now); diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java index 2a78ff562..a7bd4ca9f 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java @@ -54,7 +54,6 @@ import org.joda.time.DateTime; * transfer is automatically approved. Within that window, this flow allows the losing client to * explicitly approve the transfer request, which then becomes effective immediately. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java index dfc43776b..80f2b9678 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java @@ -54,7 +54,6 @@ import org.joda.time.DateTime; * transfer is automatically approved. Within that window, this flow allows the gaining client to * withdraw the transfer request. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java index 06287eeb2..63b9a7d3a 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java @@ -53,7 +53,6 @@ import org.joda.time.DateTime; * transfer is automatically approved. Within that window, this flow allows the losing client to * reject the transfer request. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java b/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java index fd98af00e..165ef00f8 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java @@ -63,7 +63,6 @@ import org.joda.time.Duration; * by the losing registrar or rejected, and the gaining registrar can also cancel the transfer * request. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java b/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java index 1f2d08cc7..ebbd81b2c 100644 --- a/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java +++ b/core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java @@ -55,7 +55,6 @@ import org.joda.time.DateTime; /** * An EPP flow that updates a contact. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java b/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java index b5392f133..bc5baa6ff 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java @@ -139,7 +139,6 @@ import org.joda.time.Duration; * google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException} * @error {@link * google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException} - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException} * @error {@link ResourceAlreadyExistsForThisClientException} * @error {@link ResourceCreateContentionException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java b/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java index 60b673e7f..90ed74a0e 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java @@ -103,7 +103,6 @@ import org.joda.time.Duration; /** * An EPP flow that deletes a domain. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.EppException.UnimplementedExtensionException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java b/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java index 09fa4e8fc..d5ea48f12 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java @@ -95,7 +95,6 @@ import org.joda.time.Duration; * longer than 10 years unless it comes in at the exact millisecond that the domain would have * expired. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java b/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java index fac35b7c4..17aaba76c 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java @@ -94,7 +94,6 @@ import org.joda.time.DateTime; * restored to a single year expiration starting at the restore time, regardless of what the * original expiration time was. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.EppException.UnimplementedExtensionException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java index 9199fac2f..d8cedfb78 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java @@ -78,7 +78,6 @@ import org.joda.time.DateTime; * timestamps such that they only would become active when the transfer period passed. In this flow, * those speculative objects are deleted and replaced with new ones with the correct approval time. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java index 19d430ea9..102583129 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java @@ -65,7 +65,6 @@ import org.joda.time.DateTime; * timestamps such that they only would become active when the transfer period passed. In this flow, * those speculative objects are deleted. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java index ed39bf274..9055bbdc2 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java @@ -67,7 +67,6 @@ import org.joda.time.DateTime; * timestamps such that they only would become active when the transfer period passed. In this flow, * those speculative objects are deleted. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java b/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java index 913e32515..970fe90dd 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java @@ -94,7 +94,6 @@ import org.joda.time.DateTime; * rejection or cancellation of the request, they will be deleted (and in the approval case, * replaced with new ones with the correct approval time). * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException} * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException} diff --git a/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java b/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java index c7a1bc850..2de0c3aaa 100644 --- a/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java +++ b/core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java @@ -101,7 +101,6 @@ import org.joda.time.DateTime; * superuser. As such, adding or removing these statuses incurs a billing event. There will be only * one charge per update, even if several such statuses are updated at once. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.EppException.UnimplementedExtensionException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException} diff --git a/core/src/main/java/google/registry/flows/host/HostCreateFlow.java b/core/src/main/java/google/registry/flows/host/HostCreateFlow.java index 560c33e54..22bac414d 100644 --- a/core/src/main/java/google/registry/flows/host/HostCreateFlow.java +++ b/core/src/main/java/google/registry/flows/host/HostCreateFlow.java @@ -65,7 +65,6 @@ import org.joda.time.DateTime; * hosts cannot have any. This flow allows creating a host name, and if necessary enqueues tasks to * update DNS. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.IpAddressVersionMismatchException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link ResourceAlreadyExistsForThisClientException} diff --git a/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java b/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java index babdc068b..2299fb6d6 100644 --- a/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java +++ b/core/src/main/java/google/registry/flows/host/HostDeleteFlow.java @@ -21,7 +21,6 @@ import static google.registry.flows.ResourceFlowUtils.verifyNoDisallowedStatuses import static google.registry.flows.ResourceFlowUtils.verifyResourceOwnership; import static google.registry.flows.host.HostFlowUtils.validateHostName; import static google.registry.model.eppoutput.Result.Code.SUCCESS; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import com.google.common.collect.ImmutableSet; @@ -55,7 +54,6 @@ import org.joda.time.DateTime; * references to the host before the deletion is allowed to proceed. A poll message will be written * with the success or failure message when the process is complete. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException} @@ -93,7 +91,6 @@ public final class HostDeleteFlow implements TransactionalFlow { extensionManager.register(MetadataExtension.class); validateRegistrarIsLoggedIn(registrarId); extensionManager.validate(); - assertAsyncActionsAreAllowed(); DateTime now = tm().getTransactionTime(); validateHostName(targetId); checkLinkedDomains(targetId, now, HostResource.class); diff --git a/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java b/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java index a396b564d..5921cecc8 100644 --- a/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java +++ b/core/src/main/java/google/registry/flows/host/HostUpdateFlow.java @@ -28,7 +28,6 @@ import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomain import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomainOwnership; import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey; import static google.registry.model.reporting.HistoryEntry.Type.HOST_UPDATE; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.util.CollectionUtils.isNullOrEmpty; @@ -79,7 +78,6 @@ import org.joda.time.DateTime; * when it is renamed from external to internal at least one must be added. If the host is renamed * or IP addresses are added, tasks are enqueued to update DNS accordingly. * - * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException} * @error {@link google.registry.flows.FlowUtils.NotLoggedInException} * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException} * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException} @@ -137,9 +135,6 @@ public final class HostUpdateFlow implements TransactionalFlow { validateHostName(targetId); HostResource existingHost = loadAndVerifyExistence(HostResource.class, targetId, now); boolean isHostRename = suppliedNewHostName != null; - if (isHostRename) { - assertAsyncActionsAreAllowed(); - } String oldHostName = targetId; String newHostName = firstNonNull(suppliedNewHostName, oldHostName); DomainBase oldSuperordinateDomain = diff --git a/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java b/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java index 37ec23713..dab737f53 100644 --- a/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java +++ b/core/src/main/java/google/registry/mapreduce/inputs/EppResourceInputs.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; import google.registry.model.EppResource; import google.registry.model.ImmutableObject; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.index.EppResourceIndex; /** @@ -30,6 +31,7 @@ import google.registry.model.index.EppResourceIndex; *

The inputs provided by this class are not deletion-aware and do not project the resources * forward in time. That is the responsibility of mappers that use these inputs. */ +@DeleteAfterMigration public final class EppResourceInputs { private EppResourceInputs() {} diff --git a/core/src/main/java/google/registry/model/billing/BillingEvent.java b/core/src/main/java/google/registry/model/billing/BillingEvent.java index 5906a9dcb..e634d1359 100644 --- a/core/src/main/java/google/registry/model/billing/BillingEvent.java +++ b/core/src/main/java/google/registry/model/billing/BillingEvent.java @@ -46,8 +46,6 @@ import google.registry.model.domain.DomainHistory; import google.registry.model.domain.GracePeriod; import google.registry.model.domain.rgp.GracePeriodStatus; import google.registry.model.domain.token.AllocationToken; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.model.transfer.TransferData.TransferServerApproveEntity; import google.registry.persistence.BillingVKey.BillingEventVKey; import google.registry.persistence.BillingVKey.BillingRecurrenceVKey; @@ -347,7 +345,7 @@ public abstract class BillingEvent extends ImmutableObject }) @AttributeOverride(name = "id", column = @Column(name = "billing_event_id")) @WithLongVKey(compositeKey = true) - public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity { + public static class OneTime extends BillingEvent { /** The billable value. */ @Type(type = JodaMoneyType.TYPE_NAME) @@ -559,7 +557,7 @@ public abstract class BillingEvent extends ImmutableObject }) @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id")) @WithLongVKey(compositeKey = true) - public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity { + public static class Recurring extends BillingEvent { /** * The billing event recurs every year between {@link #eventTime} and this time on the @@ -696,7 +694,7 @@ public abstract class BillingEvent extends ImmutableObject }) @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id")) @WithLongVKey(compositeKey = true) - public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity { + public static class Cancellation extends BillingEvent { /** The billing time of the charge that is being cancelled. */ @Index @@ -819,7 +817,7 @@ public abstract class BillingEvent extends ImmutableObject @ReportedOn @Entity @WithLongVKey(compositeKey = true) - public static class Modification extends BillingEvent implements DatastoreOnlyEntity { + public static class Modification extends BillingEvent { /** The change in cost that should be applied to the original billing event. */ Money cost; diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java b/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java index 75ef8b14b..7fc845ab2 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java @@ -16,7 +16,6 @@ package google.registry.model.bulkquery; import google.registry.model.domain.DomainBase; import google.registry.model.domain.DomainContent; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import javax.persistence.Access; @@ -34,7 +33,7 @@ import javax.persistence.Entity; @Entity(name = "Domain") @WithStringVKey @Access(AccessType.FIELD) -public class DomainBaseLite extends DomainContent implements SqlOnlyEntity { +public class DomainBaseLite extends DomainContent { @Override @javax.persistence.Id diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java index 796d104a7..f50f41630 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java @@ -17,7 +17,6 @@ package google.registry.model.bulkquery; import com.google.common.base.Objects; import google.registry.model.domain.DomainHistory.DomainHistoryId; import google.registry.model.host.HostResource; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.VKey; import java.io.Serializable; import javax.persistence.Access; @@ -33,7 +32,7 @@ import javax.persistence.IdClass; @Entity @Access(AccessType.FIELD) @IdClass(DomainHistoryHost.class) -public class DomainHistoryHost implements Serializable, SqlOnlyEntity { +public class DomainHistoryHost implements Serializable { @Id private Long domainHistoryHistoryRevisionId; @Id private String domainHistoryDomainRepoId; diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java index 4891031b7..11af0cde5 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java @@ -20,7 +20,6 @@ import google.registry.model.domain.DomainContent; import google.registry.model.domain.DomainHistory; import google.registry.model.domain.DomainHistory.DomainHistoryId; import google.registry.model.domain.Period; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import javax.annotation.Nullable; @@ -48,7 +47,7 @@ import javax.persistence.PostLoad; @Entity(name = "DomainHistory") @Access(AccessType.FIELD) @IdClass(DomainHistoryId.class) -public class DomainHistoryLite extends HistoryEntry implements SqlOnlyEntity { +public class DomainHistoryLite extends HistoryEntry { // Store DomainContent instead of DomainBase so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects diff --git a/core/src/main/java/google/registry/model/bulkquery/DomainHost.java b/core/src/main/java/google/registry/model/bulkquery/DomainHost.java index 8055b5f4c..df427da90 100644 --- a/core/src/main/java/google/registry/model/bulkquery/DomainHost.java +++ b/core/src/main/java/google/registry/model/bulkquery/DomainHost.java @@ -16,7 +16,6 @@ package google.registry.model.bulkquery; import com.google.common.base.Objects; import google.registry.model.host.HostResource; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.VKey; import java.io.Serializable; import javax.persistence.Access; @@ -29,7 +28,7 @@ import javax.persistence.IdClass; @Entity @Access(AccessType.FIELD) @IdClass(DomainHost.class) -public class DomainHost implements Serializable, SqlOnlyEntity { +public class DomainHost implements Serializable { @Id private String domainRepoId; diff --git a/core/src/main/java/google/registry/model/common/Cursor.java b/core/src/main/java/google/registry/model/common/Cursor.java index 2aca2ad80..7f28adc54 100644 --- a/core/src/main/java/google/registry/model/common/Cursor.java +++ b/core/src/main/java/google/registry/model/common/Cursor.java @@ -31,7 +31,6 @@ import google.registry.model.UnsafeSerializable; import google.registry.model.UpdateAutoTimestamp; import google.registry.model.annotations.InCrossTld; import google.registry.model.common.Cursor.CursorId; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.tld.Registry; import google.registry.persistence.VKey; import java.util.List; @@ -53,7 +52,7 @@ import org.joda.time.DateTime; @javax.persistence.Entity @IdClass(CursorId.class) @InCrossTld -public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity, UnsafeSerializable { +public class Cursor extends ImmutableObject implements UnsafeSerializable { /** The scope of a global cursor. A global cursor is a cursor that is not specific to one tld. */ public static final String GLOBAL = "GLOBAL"; diff --git a/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java b/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java index bafe6d7bf..a99a873a1 100644 --- a/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java +++ b/core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java @@ -27,7 +27,6 @@ import google.registry.config.RegistryEnvironment; import google.registry.model.CacheUtils; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.common.TimedTransitionProperty.TimedTransition; -import google.registry.model.replay.SqlOnlyEntity; import java.time.Duration; import java.util.Arrays; import javax.persistence.Entity; @@ -42,7 +41,7 @@ import org.joda.time.DateTime; */ @DeleteAfterMigration @Entity -public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements SqlOnlyEntity { +public class DatabaseMigrationStateSchedule extends CrossTldSingleton { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -218,7 +217,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements MigrationState.DATASTORE_ONLY, "migrationTransitionMap must start with DATASTORE_ONLY"); validateTransitionAtCurrentTime(transitions); - jpaTm().putIgnoringReadOnlyWithoutBackup(new DatabaseMigrationStateSchedule(transitions)); + jpaTm().putWithoutBackup(new DatabaseMigrationStateSchedule(transitions)); CACHE.invalidateAll(); } diff --git a/core/src/main/java/google/registry/model/common/EntityGroupRoot.java b/core/src/main/java/google/registry/model/common/EntityGroupRoot.java index 6009ad454..bbd4f273c 100644 --- a/core/src/main/java/google/registry/model/common/EntityGroupRoot.java +++ b/core/src/main/java/google/registry/model/common/EntityGroupRoot.java @@ -20,7 +20,6 @@ import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Id; import google.registry.model.BackupGroupRoot; import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.replay.DatastoreOnlyEntity; import javax.annotation.Nullable; /** @@ -39,7 +38,7 @@ import javax.annotation.Nullable; */ @Entity @DeleteAfterMigration -public class EntityGroupRoot extends BackupGroupRoot implements DatastoreOnlyEntity { +public class EntityGroupRoot extends BackupGroupRoot { @SuppressWarnings("unused") @Id diff --git a/core/src/main/java/google/registry/model/contact/ContactHistory.java b/core/src/main/java/google/registry/model/contact/ContactHistory.java index d59d5fe03..1683133fa 100644 --- a/core/src/main/java/google/registry/model/contact/ContactHistory.java +++ b/core/src/main/java/google/registry/model/contact/ContactHistory.java @@ -20,8 +20,6 @@ import google.registry.model.EppResource; import google.registry.model.ImmutableObject; import google.registry.model.UnsafeSerializable; import google.registry.model.contact.ContactHistory.ContactHistoryId; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import java.io.Serializable; @@ -58,7 +56,7 @@ import javax.persistence.PostLoad; @EntitySubclass @Access(AccessType.FIELD) @IdClass(ContactHistoryId.class) -public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable { +public class ContactHistory extends HistoryEntry implements UnsafeSerializable { // Store ContactBase instead of ContactResource so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects @@ -128,12 +126,6 @@ public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSer } } - // In Datastore, save as a HistoryEntry object regardless of this object's type - @Override - public Optional toDatastoreEntity() { - return Optional.of(asHistoryEntry()); - } - /** Class to represent the composite primary key of {@link ContactHistory} entity. */ public static class ContactHistoryId extends ImmutableObject implements Serializable { diff --git a/core/src/main/java/google/registry/model/contact/ContactResource.java b/core/src/main/java/google/registry/model/contact/ContactResource.java index 2c31c98a5..59c98f772 100644 --- a/core/src/main/java/google/registry/model/contact/ContactResource.java +++ b/core/src/main/java/google/registry/model/contact/ContactResource.java @@ -19,7 +19,6 @@ import com.googlecode.objectify.annotation.Entity; import google.registry.model.EppResource.ForeignKeyedEppResource; import google.registry.model.annotations.ExternalMessagingName; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import javax.persistence.Access; @@ -46,8 +45,7 @@ import org.joda.time.DateTime; @ExternalMessagingName("contact") @WithStringVKey @Access(AccessType.FIELD) -public class ContactResource extends ContactBase - implements DatastoreAndSqlEntity, ForeignKeyedEppResource { +public class ContactResource extends ContactBase implements ForeignKeyedEppResource { @Override public VKey createVKey() { diff --git a/core/src/main/java/google/registry/model/domain/DomainBase.java b/core/src/main/java/google/registry/model/domain/DomainBase.java index 0950daddb..c94ca4987 100644 --- a/core/src/main/java/google/registry/model/domain/DomainBase.java +++ b/core/src/main/java/google/registry/model/domain/DomainBase.java @@ -21,7 +21,6 @@ import google.registry.model.annotations.ExternalMessagingName; import google.registry.model.annotations.ReportedOn; import google.registry.model.domain.secdns.DelegationSignerData; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import java.util.Set; @@ -74,8 +73,7 @@ import org.joda.time.DateTime; @WithStringVKey @ExternalMessagingName("domain") @Access(AccessType.FIELD) -public class DomainBase extends DomainContent - implements DatastoreAndSqlEntity, ForeignKeyedEppResource { +public class DomainBase extends DomainContent implements ForeignKeyedEppResource { @Override @javax.persistence.Id diff --git a/core/src/main/java/google/registry/model/domain/DomainHistory.java b/core/src/main/java/google/registry/model/domain/DomainHistory.java index 3cb3e633a..34cb44fac 100644 --- a/core/src/main/java/google/registry/model/domain/DomainHistory.java +++ b/core/src/main/java/google/registry/model/domain/DomainHistory.java @@ -27,8 +27,6 @@ import google.registry.model.domain.GracePeriod.GracePeriodHistory; import google.registry.model.domain.secdns.DelegationSignerData; import google.registry.model.domain.secdns.DomainDsDataHistory; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.DomainTransactionRecord; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; @@ -80,7 +78,7 @@ import org.hibernate.Hibernate; @EntitySubclass @Access(AccessType.FIELD) @IdClass(DomainHistoryId.class) -public class DomainHistory extends HistoryEntry implements SqlEntity { +public class DomainHistory extends HistoryEntry { // Store DomainContent instead of DomainBase so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects @@ -295,12 +293,6 @@ public class DomainHistory extends HistoryEntry implements SqlEntity { } } - // In Datastore, save as a HistoryEntry object regardless of this object's type - @Override - public Optional toDatastoreEntity() { - return Optional.of(asHistoryEntry()); - } - private static void fillAuxiliaryFieldsFromDomain(DomainHistory domainHistory) { if (domainHistory.domainContent != null) { domainHistory.nsHosts = nullToEmptyImmutableCopy(domainHistory.domainContent.nsHosts); diff --git a/core/src/main/java/google/registry/model/domain/GracePeriod.java b/core/src/main/java/google/registry/model/domain/GracePeriod.java index 486f42880..9db7171f6 100644 --- a/core/src/main/java/google/registry/model/domain/GracePeriod.java +++ b/core/src/main/java/google/registry/model/domain/GracePeriod.java @@ -24,8 +24,6 @@ import google.registry.model.billing.BillingEvent; import google.registry.model.billing.BillingEvent.Recurring; import google.registry.model.domain.DomainHistory.DomainHistoryId; import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.BillingVKey.BillingEventVKey; import google.registry.persistence.BillingVKey.BillingRecurrenceVKey; import google.registry.persistence.VKey; @@ -52,7 +50,7 @@ import org.joda.time.DateTime; @Index(columnList = "billing_event_id"), @Index(columnList = "billing_recurrence_id") }) -public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntity { +public class GracePeriod extends GracePeriodBase { @Id @Access(AccessType.PROPERTY) @@ -197,7 +195,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit /** Entity class to represent a historic {@link GracePeriod}. */ @Entity(name = "GracePeriodHistory") @Table(indexes = @Index(columnList = "domainRepoId")) - public static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity { + public static class GracePeriodHistory extends GracePeriodBase { @Id Long gracePeriodHistoryRevisionId; /** ID for the associated {@link DomainHistory} entity. */ diff --git a/core/src/main/java/google/registry/model/domain/RegistryLock.java b/core/src/main/java/google/registry/model/domain/RegistryLock.java index cf1adb511..daedb8e9f 100644 --- a/core/src/main/java/google/registry/model/domain/RegistryLock.java +++ b/core/src/main/java/google/registry/model/domain/RegistryLock.java @@ -23,7 +23,6 @@ import google.registry.model.Buildable; import google.registry.model.CreateAutoTimestamp; import google.registry.model.ImmutableObject; import google.registry.model.UpdateAutoTimestamp; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.util.DateTimeUtils; import java.time.ZonedDateTime; import java.util.Optional; @@ -76,7 +75,7 @@ import org.joda.time.Duration; @Index(name = "idx_registry_lock_verification_code", columnList = "verificationCode"), @Index(name = "idx_registry_lock_registrar_id", columnList = "registrarId") }) -public final class RegistryLock extends ImmutableObject implements Buildable, SqlOnlyEntity { +public final class RegistryLock extends ImmutableObject implements Buildable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java b/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java index 4d1e646a3..bcdf392dc 100644 --- a/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java +++ b/core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java @@ -19,7 +19,6 @@ import static google.registry.model.IdService.allocateId; import google.registry.model.UnsafeSerializable; import google.registry.model.domain.DomainHistory; import google.registry.model.domain.DomainHistory.DomainHistoryId; -import google.registry.model.replay.SqlOnlyEntity; import javax.persistence.Access; import javax.persistence.AccessType; import javax.persistence.Column; @@ -28,8 +27,7 @@ import javax.persistence.Id; /** Entity class to represent a historic {@link DelegationSignerData}. */ @Entity -public class DomainDsDataHistory extends DomainDsDataBase - implements SqlOnlyEntity, UnsafeSerializable { +public class DomainDsDataHistory extends DomainDsDataBase implements UnsafeSerializable { @Id Long dsDataHistoryRevisionId; diff --git a/core/src/main/java/google/registry/model/domain/token/AllocationToken.java b/core/src/main/java/google/registry/model/domain/token/AllocationToken.java index aab73084b..5c96f25b1 100644 --- a/core/src/main/java/google/registry/model/domain/token/AllocationToken.java +++ b/core/src/main/java/google/registry/model/domain/token/AllocationToken.java @@ -47,7 +47,6 @@ import google.registry.model.billing.BillingEvent.RenewalPriceBehavior; import google.registry.model.common.TimedTransitionProperty; import google.registry.model.common.TimedTransitionProperty.TimeMapper; import google.registry.model.common.TimedTransitionProperty.TimedTransition; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.DomainHistoryVKey; import google.registry.persistence.VKey; @@ -80,7 +79,7 @@ import org.joda.time.DateTime; @javax.persistence.Index(columnList = "tokenType"), @javax.persistence.Index(columnList = "redemption_domain_repo_id") }) -public class AllocationToken extends BackupGroupRoot implements Buildable, DatastoreAndSqlEntity { +public class AllocationToken extends BackupGroupRoot implements Buildable { // Promotions should only move forward, and ENDED / CANCELLED are terminal states. private static final ImmutableMultimap VALID_TOKEN_STATUS_TRANSITIONS = diff --git a/core/src/main/java/google/registry/model/host/HostHistory.java b/core/src/main/java/google/registry/model/host/HostHistory.java index ec9b541bf..d7c3a879e 100644 --- a/core/src/main/java/google/registry/model/host/HostHistory.java +++ b/core/src/main/java/google/registry/model/host/HostHistory.java @@ -20,8 +20,6 @@ import google.registry.model.EppResource; import google.registry.model.ImmutableObject; import google.registry.model.UnsafeSerializable; import google.registry.model.host.HostHistory.HostHistoryId; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import java.io.Serializable; @@ -59,7 +57,7 @@ import javax.persistence.PostLoad; @EntitySubclass @Access(AccessType.FIELD) @IdClass(HostHistoryId.class) -public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable { +public class HostHistory extends HistoryEntry implements UnsafeSerializable { // Store HostBase instead of HostResource so we don't pick up its @Id // Nullable for the sake of pre-Registry-3.0 history objects @@ -128,12 +126,6 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial } } - // In Datastore, save as a HistoryEntry object regardless of this object's type - @Override - public Optional toDatastoreEntity() { - return Optional.of(asHistoryEntry()); - } - /** Class to represent the composite primary key of {@link HostHistory} entity. */ public static class HostHistoryId extends ImmutableObject implements Serializable { diff --git a/core/src/main/java/google/registry/model/host/HostResource.java b/core/src/main/java/google/registry/model/host/HostResource.java index aa952b5af..7436d8086 100644 --- a/core/src/main/java/google/registry/model/host/HostResource.java +++ b/core/src/main/java/google/registry/model/host/HostResource.java @@ -19,7 +19,6 @@ import com.googlecode.objectify.annotation.Entity; import google.registry.model.EppResource.ForeignKeyedEppResource; import google.registry.model.annotations.ExternalMessagingName; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.WithStringVKey; import javax.persistence.Access; @@ -55,8 +54,7 @@ import javax.persistence.AccessType; @ExternalMessagingName("host") @WithStringVKey @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property) -public class HostResource extends HostBase - implements DatastoreAndSqlEntity, ForeignKeyedEppResource { +public class HostResource extends HostBase implements ForeignKeyedEppResource { @Override @javax.persistence.Id diff --git a/core/src/main/java/google/registry/model/index/EppResourceIndex.java b/core/src/main/java/google/registry/model/index/EppResourceIndex.java index 5a577d9ba..8b78a3bc5 100644 --- a/core/src/main/java/google/registry/model/index/EppResourceIndex.java +++ b/core/src/main/java/google/registry/model/index/EppResourceIndex.java @@ -26,14 +26,13 @@ import google.registry.model.BackupGroupRoot; import google.registry.model.EppResource; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.persistence.VKey; /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */ @ReportedOn @Entity @DeleteAfterMigration -public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEntity { +public class EppResourceIndex extends BackupGroupRoot { @Id String id; diff --git a/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java b/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java index ad41ff035..2f33d4299 100644 --- a/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java +++ b/core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java @@ -25,13 +25,12 @@ import google.registry.model.EppResource; import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.VirtualEntity; -import google.registry.model.replay.DatastoreOnlyEntity; /** A virtual entity to represent buckets to which EppResourceIndex objects are randomly added. */ @Entity @VirtualEntity @DeleteAfterMigration -public class EppResourceIndexBucket extends ImmutableObject implements DatastoreOnlyEntity { +public class EppResourceIndexBucket extends ImmutableObject { @SuppressWarnings("unused") @Id diff --git a/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java b/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java index 0b7e9ee3a..2ac3b6907 100644 --- a/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java +++ b/core/src/main/java/google/registry/model/index/ForeignKeyIndex.java @@ -50,7 +50,6 @@ import google.registry.model.annotations.ReportedOn; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DomainBase; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.persistence.VKey; import google.registry.persistence.transaction.CriteriaQueryBuilder; import google.registry.persistence.transaction.JpaTransactionManager; @@ -74,20 +73,17 @@ public abstract class ForeignKeyIndex extends BackupGroup /** The {@link ForeignKeyIndex} type for {@link ContactResource} entities. */ @ReportedOn @Entity - public static class ForeignKeyContactIndex extends ForeignKeyIndex - implements DatastoreOnlyEntity {} + public static class ForeignKeyContactIndex extends ForeignKeyIndex {} /** The {@link ForeignKeyIndex} type for {@link DomainBase} entities. */ @ReportedOn @Entity - public static class ForeignKeyDomainIndex extends ForeignKeyIndex - implements DatastoreOnlyEntity {} + public static class ForeignKeyDomainIndex extends ForeignKeyIndex {} /** The {@link ForeignKeyIndex} type for {@link HostResource} entities. */ @ReportedOn @Entity - public static class ForeignKeyHostIndex extends ForeignKeyIndex - implements DatastoreOnlyEntity {} + public static class ForeignKeyHostIndex extends ForeignKeyIndex {} private static final ImmutableBiMap< Class, Class>> diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java b/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java index c8e53747f..9221304c9 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogBucket.java @@ -34,7 +34,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import google.registry.util.NonFinalForTesting; import java.util.Random; import java.util.function.Supplier; @@ -53,7 +52,7 @@ import org.joda.time.DateTime; @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogBucket extends ImmutableObject implements Buildable, DatastoreOnlyEntity { +public class CommitLogBucket extends ImmutableObject implements Buildable { /** * Ranges from 1 to {@link RegistryConfig#getCommitLogBucketCount()}, inclusive; starts at 1 since diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java index 867f5f804..55cc57184 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java @@ -28,7 +28,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -47,7 +46,7 @@ import org.joda.time.DateTime; @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogCheckpoint extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogCheckpoint extends ImmutableObject { /** Shared singleton parent entity for commit log checkpoints. */ @Parent diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java index e173f61d6..b25b9ebea 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java @@ -24,14 +24,13 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import org.joda.time.DateTime; /** Singleton parent entity for all commit log checkpoints. */ @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogCheckpointRoot extends ImmutableObject { public static final long SINGLETON_ID = 1; // There is always exactly one of these. diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java b/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java index 8ad8248f7..78b1528e6 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogManifest.java @@ -26,7 +26,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; import java.util.LinkedHashSet; import java.util.Set; import org.joda.time.DateTime; @@ -41,7 +40,7 @@ import org.joda.time.DateTime; @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogManifest extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogManifest extends ImmutableObject { /** Commit log manifests are parented on a random bucket. */ @Parent diff --git a/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java b/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java index 87df17d7e..66f02aa38 100644 --- a/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java +++ b/core/src/main/java/google/registry/model/ofy/CommitLogMutation.java @@ -30,13 +30,12 @@ import google.registry.model.ImmutableObject; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreOnlyEntity; /** Representation of a saved entity in a {@link CommitLogManifest} (not deletes). */ @Entity @NotBackedUp(reason = Reason.COMMIT_LOGS) @DeleteAfterMigration -public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyEntity { +public class CommitLogMutation extends ImmutableObject { /** The manifest this belongs to. */ @Parent diff --git a/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java b/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java index e8187d2f3..94ab71043 100644 --- a/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java +++ b/core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java @@ -38,8 +38,6 @@ import google.registry.model.annotations.InCrossTld; import google.registry.model.contact.ContactHistory; import google.registry.model.domain.DomainHistory; import google.registry.model.host.HostHistory; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.VKey; import google.registry.persistence.transaction.QueryComposer; @@ -357,28 +355,6 @@ public class DatastoreTransactionManager implements TransactionManager { return true; } - @Override - public void putIgnoringReadOnlyWithoutBackup(Object entity) { - syncIfTransactionless( - getOfy().saveIgnoringReadOnlyWithoutBackup().entities(toDatastoreEntity(entity))); - } - - @Override - public void deleteIgnoringReadOnlyWithoutBackup(VKey key) { - syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithoutBackup().key(key.getOfyKey())); - } - - /** Performs the write ignoring read-only restrictions and also writes commit logs. */ - public void putIgnoringReadOnlyWithBackup(Object entity) { - syncIfTransactionless( - getOfy().saveIgnoringReadOnlyWithBackup().entities(toDatastoreEntity(entity))); - } - - /** Performs the delete ignoring read-only restrictions and also writes commit logs. */ - public void deleteIgnoringReadOnlyWithBackup(VKey key) { - syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithBackup().key(key.getOfyKey())); - } - /** * Executes the given {@link Result} instance synchronously if not in a transaction. * @@ -413,22 +389,14 @@ public class DatastoreTransactionManager implements TransactionManager { return toSqlEntity(getOfy().load().key(key.getOfyKey()).now()); } - /** - * Converts a possible {@link SqlEntity} to a {@link DatastoreEntity}. - * - *

One example is that this would convert a {@link DomainHistory} to a {@link HistoryEntry}. - */ + /** Converts a possible {@link HistoryEntry} child to a {@link HistoryEntry}. */ private static Object toDatastoreEntity(@Nullable Object obj) { - if (obj instanceof SqlEntity) { - Optional possibleDatastoreEntity = ((SqlEntity) obj).toDatastoreEntity(); - if (possibleDatastoreEntity.isPresent()) { - return possibleDatastoreEntity.get(); - } + if (obj instanceof HistoryEntry) { + return ((HistoryEntry) obj).asHistoryEntry(); } return obj; } - /** Converts many possible {@link SqlEntity} objects to {@link DatastoreEntity} objects. */ private static ImmutableList toDatastoreEntities(ImmutableCollection collection) { return collection.stream() .map(DatastoreTransactionManager::toDatastoreEntity) @@ -436,21 +404,15 @@ public class DatastoreTransactionManager implements TransactionManager { } /** - * Converts an object to the corresponding {@link SqlEntity} if necessary and possible. + * Converts an object to the corresponding child {@link HistoryEntry} if necessary and possible. * *

This should be used when returning objects from Datastore to make sure they reflect the most * recent type of the object in question. */ @SuppressWarnings("unchecked") public static T toSqlEntity(@Nullable T obj) { - // NB: The Key of the object in question may not necessarily be the resulting class that we - // wish to have. For example, because all *History classes are @EntitySubclasses, their Keys - // will have type HistoryEntry -- even if you create them based off the *History class. - if (obj instanceof DatastoreEntity && !(obj instanceof SqlEntity)) { - Optional possibleSqlEntity = ((DatastoreEntity) obj).toSqlEntity(); - if (possibleSqlEntity.isPresent()) { - return (T) possibleSqlEntity.get(); - } + if (obj instanceof HistoryEntry) { + return (T) ((HistoryEntry) obj).toChildHistoryEntity(); } return obj; } diff --git a/core/src/main/java/google/registry/model/ofy/Ofy.java b/core/src/main/java/google/registry/model/ofy/Ofy.java index eafa8c1d6..9f56f864d 100644 --- a/core/src/main/java/google/registry/model/ofy/Ofy.java +++ b/core/src/main/java/google/registry/model/ofy/Ofy.java @@ -19,7 +19,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Maps.uniqueIndex; import static com.googlecode.objectify.ObjectifyService.ofy; import static google.registry.config.RegistryConfig.getBaseOfyRetryDuration; -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; import static google.registry.util.CollectionUtils.union; import com.google.appengine.api.datastore.DatastoreFailureException; @@ -134,7 +133,6 @@ public class Ofy { *

We only allow this in transactions so commit logs can be written in tandem with the delete. */ public Deleter delete() { - assertNotReadOnlyMode(); return deleteIgnoringReadOnlyWithBackup(); } @@ -144,7 +142,6 @@ public class Ofy { *

No backups get written. */ public Deleter deleteWithoutBackup() { - assertNotReadOnlyMode(); return deleteIgnoringReadOnlyWithoutBackup(); } @@ -155,7 +152,6 @@ public class Ofy { *

We only allow this in transactions so commit logs can be written in tandem with the save. */ public Saver save() { - assertNotReadOnlyMode(); return saveIgnoringReadOnlyWithBackup(); } @@ -165,7 +161,6 @@ public class Ofy { *

No backups get written. */ public Saver saveWithoutBackup() { - assertNotReadOnlyMode(); return saveIgnoringReadOnlyWithoutBackup(); } diff --git a/core/src/main/java/google/registry/model/poll/PollMessage.java b/core/src/main/java/google/registry/model/poll/PollMessage.java index 7491c4551..3fce7a691 100644 --- a/core/src/main/java/google/registry/model/poll/PollMessage.java +++ b/core/src/main/java/google/registry/model/poll/PollMessage.java @@ -44,7 +44,6 @@ import google.registry.model.host.HostResource; import google.registry.model.poll.PendingActionNotificationResponse.ContactPendingActionNotificationResponse; import google.registry.model.poll.PendingActionNotificationResponse.DomainPendingActionNotificationResponse; import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.reporting.HistoryEntry; import google.registry.model.transfer.TransferData.TransferServerApproveEntity; import google.registry.model.transfer.TransferResponse; @@ -99,7 +98,7 @@ import org.joda.time.DateTime; @javax.persistence.Index(columnList = "eventTime") }) public abstract class PollMessage extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity, UnsafeSerializable { + implements Buildable, TransferServerApproveEntity, UnsafeSerializable { /** Entity id. */ @Id diff --git a/core/src/main/java/google/registry/model/rde/RdeRevision.java b/core/src/main/java/google/registry/model/rde/RdeRevision.java index 659c8ee48..0a2d06bd8 100644 --- a/core/src/main/java/google/registry/model/rde/RdeRevision.java +++ b/core/src/main/java/google/registry/model/rde/RdeRevision.java @@ -27,7 +27,6 @@ import com.googlecode.objectify.annotation.Ignore; import google.registry.model.BackupGroupRoot; import google.registry.model.ImmutableObject; import google.registry.model.rde.RdeRevision.RdeRevisionId; -import google.registry.model.replay.NonReplicatedEntity; import google.registry.persistence.VKey; import google.registry.persistence.converter.LocalDateConverter; import java.io.Serializable; @@ -51,7 +50,7 @@ import org.joda.time.LocalDate; @Entity @javax.persistence.Entity @IdClass(RdeRevisionId.class) -public final class RdeRevision extends BackupGroupRoot implements NonReplicatedEntity { +public final class RdeRevision extends BackupGroupRoot { /** String triplet of tld, date, and mode, e.g. {@code soy_2015-09-01_full}. */ @Id @Transient String id; diff --git a/core/src/main/java/google/registry/model/registrar/Registrar.java b/core/src/main/java/google/registry/model/registrar/Registrar.java index efb71ff1d..70c5e3997 100644 --- a/core/src/main/java/google/registry/model/registrar/Registrar.java +++ b/core/src/main/java/google/registry/model/registrar/Registrar.java @@ -79,7 +79,6 @@ import google.registry.model.annotations.InCrossTld; import google.registry.model.annotations.ReportedOn; import google.registry.model.common.EntityGroupRoot; import google.registry.model.registrar.Registrar.BillingAccountEntry.CurrencyMapper; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.tld.Registry; import google.registry.model.tld.Registry.TldType; import google.registry.persistence.VKey; @@ -119,7 +118,7 @@ import org.joda.time.DateTime; }) @InCrossTld public class Registrar extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable { + implements Buildable, Jsonifiable, UnsafeSerializable { /** Represents the type of a registrar entity. */ public enum Type { diff --git a/core/src/main/java/google/registry/model/registrar/RegistrarContact.java b/core/src/main/java/google/registry/model/registrar/RegistrarContact.java index d4aefc041..bbab8b6ba 100644 --- a/core/src/main/java/google/registry/model/registrar/RegistrarContact.java +++ b/core/src/main/java/google/registry/model/registrar/RegistrarContact.java @@ -50,7 +50,6 @@ import google.registry.model.UnsafeSerializable; import google.registry.model.annotations.InCrossTld; import google.registry.model.annotations.ReportedOn; import google.registry.model.registrar.RegistrarContact.RegistrarPocId; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import java.io.Serializable; import java.util.Arrays; @@ -82,8 +81,7 @@ import javax.persistence.Transient; }) @IdClass(RegistrarPocId.class) @InCrossTld -public class RegistrarContact extends ImmutableObject - implements DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable { +public class RegistrarContact extends ImmutableObject implements Jsonifiable, UnsafeSerializable { @Parent @Transient Key parent; diff --git a/core/src/main/java/google/registry/model/replay/DatastoreAndSqlEntity.java b/core/src/main/java/google/registry/model/replay/DatastoreAndSqlEntity.java deleted file mode 100644 index 8fb61dc90..000000000 --- a/core/src/main/java/google/registry/model/replay/DatastoreAndSqlEntity.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** An entity that has the same Java object representation in SQL and Datastore. */ -@DeleteAfterMigration -public interface DatastoreAndSqlEntity extends DatastoreEntity, SqlEntity { - - @Override - default Optional toDatastoreEntity() { - return Optional.of(this); - } - - @Override - default Optional toSqlEntity() { - return Optional.of(this); - } -} diff --git a/core/src/main/java/google/registry/model/replay/DatastoreEntity.java b/core/src/main/java/google/registry/model/replay/DatastoreEntity.java deleted file mode 100644 index 7824455b8..000000000 --- a/core/src/main/java/google/registry/model/replay/DatastoreEntity.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** - * An object that can be stored in Datastore and serialized using Objectify's {@link - * com.googlecode.objectify.cmd.Saver#toEntity(Object)} code. - * - *

This is used when replaying {@link google.registry.model.ofy.CommitLogManifest}s to import - * transactions and data into the secondary SQL store during the first, Datastore-primary, phase of - * the migration. - */ -@DeleteAfterMigration -public interface DatastoreEntity { - - Optional toSqlEntity(); -} diff --git a/core/src/main/java/google/registry/model/replay/DatastoreOnlyEntity.java b/core/src/main/java/google/registry/model/replay/DatastoreOnlyEntity.java deleted file mode 100644 index 842e31a51..000000000 --- a/core/src/main/java/google/registry/model/replay/DatastoreOnlyEntity.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** An entity that is only stored in Datastore, that should not be replayed to SQL. */ -@DeleteAfterMigration -public interface DatastoreOnlyEntity extends DatastoreEntity { - @Override - default Optional toSqlEntity() { - return Optional.empty(); - } -} diff --git a/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java b/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java index 678e24c8b..51a1a503a 100644 --- a/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java +++ b/core/src/main/java/google/registry/model/replay/LastSqlTransaction.java @@ -27,7 +27,7 @@ import google.registry.model.annotations.DeleteAfterMigration; /** Datastore entity to keep track of the last SQL transaction imported into the datastore. */ @Entity @DeleteAfterMigration -public class LastSqlTransaction extends ImmutableObject implements DatastoreOnlyEntity { +public class LastSqlTransaction extends ImmutableObject { /** The key for this singleton. */ public static final Key KEY = Key.create(LastSqlTransaction.class, 1); diff --git a/core/src/main/java/google/registry/model/replay/NonReplicatedEntity.java b/core/src/main/java/google/registry/model/replay/NonReplicatedEntity.java deleted file mode 100644 index 7878a3132..000000000 --- a/core/src/main/java/google/registry/model/replay/NonReplicatedEntity.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** - * Represents an entity that should not participate in asynchronous replication. - * - *

We expect that this is a result of the entity being dually-written. - */ -@DeleteAfterMigration -public interface NonReplicatedEntity extends DatastoreEntity, SqlEntity { - - @Override - default Optional toDatastoreEntity() { - return Optional.empty(); - } - - @Override - default Optional toSqlEntity() { - return Optional.empty(); - } -} diff --git a/core/src/main/java/google/registry/model/replay/ReplayGap.java b/core/src/main/java/google/registry/model/replay/ReplayGap.java index 921c9ea35..a850b54e4 100644 --- a/core/src/main/java/google/registry/model/replay/ReplayGap.java +++ b/core/src/main/java/google/registry/model/replay/ReplayGap.java @@ -35,7 +35,7 @@ import org.joda.time.DateTime; @DeleteAfterMigration @NotBackedUp(reason = TRANSIENT) @Entity -public class ReplayGap extends ImmutableObject implements DatastoreOnlyEntity { +public class ReplayGap extends ImmutableObject { @Id long transactionId; // We can't use a CreateAutoTimestamp here because this ends up getting persisted in an ofy diff --git a/core/src/main/java/google/registry/model/replay/SqlEntity.java b/core/src/main/java/google/registry/model/replay/SqlEntity.java deleted file mode 100644 index ea09469f7..000000000 --- a/core/src/main/java/google/registry/model/replay/SqlEntity.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** - * An object that can be stored in Cloud SQL using {@link - * javax.persistence.EntityManager#persist(Object)} - * - *

This will be used when replaying SQL transactions into Datastore, during the second, - * SQL-primary, phase of the migration from Datastore to SQL. - */ -@DeleteAfterMigration -public interface SqlEntity { - - Optional toDatastoreEntity(); - - /** Returns this entity's primary key field(s) in a string. */ - default String getPrimaryKeyString() { - return jpaTm() - .transact( - () -> - String.format( - "%s_%s", - this.getClass().getSimpleName(), - jpaTm() - .getEntityManager() - .getEntityManagerFactory() - .getPersistenceUnitUtil() - .getIdentifier(this))); - } -} diff --git a/core/src/main/java/google/registry/model/replay/SqlOnlyEntity.java b/core/src/main/java/google/registry/model/replay/SqlOnlyEntity.java deleted file mode 100644 index ac156b758..000000000 --- a/core/src/main/java/google/registry/model/replay/SqlOnlyEntity.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Optional; - -/** An entity that is only stored in SQL, that should not be replayed to Datastore. */ -@DeleteAfterMigration -public interface SqlOnlyEntity extends SqlEntity { - @Override - default Optional toDatastoreEntity() { - return Optional.empty(); - } -} diff --git a/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java b/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java index 89f8cedf1..5ec529e3d 100644 --- a/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java +++ b/core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java @@ -25,7 +25,7 @@ import org.joda.time.DateTime; @Entity @DeleteAfterMigration -public class SqlReplayCheckpoint extends CrossTldSingleton implements SqlOnlyEntity { +public class SqlReplayCheckpoint extends CrossTldSingleton { @Column(nullable = false) private DateTime lastReplayTime; @@ -43,6 +43,6 @@ public class SqlReplayCheckpoint extends CrossTldSingleton implements SqlOnlyEnt SqlReplayCheckpoint checkpoint = new SqlReplayCheckpoint(); checkpoint.lastReplayTime = lastReplayTime; // this will overwrite the existing object due to the constant revisionId - jpaTm().putIgnoringReadOnlyWithoutBackup(checkpoint); + jpaTm().put(checkpoint); } } diff --git a/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java b/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java index 1f9ccaf8d..52969cec5 100644 --- a/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java +++ b/core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java @@ -24,7 +24,6 @@ import google.registry.model.Buildable; import google.registry.model.ImmutableObject; import google.registry.model.UnsafeSerializable; import google.registry.model.domain.DomainHistory.DomainHistoryId; -import google.registry.model.replay.DatastoreAndSqlEntity; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; @@ -47,7 +46,7 @@ import org.joda.time.DateTime; @Embed @Entity public class DomainTransactionRecord extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable { + implements Buildable, UnsafeSerializable { @Id @Ignore diff --git a/core/src/main/java/google/registry/model/reporting/HistoryEntry.java b/core/src/main/java/google/registry/model/reporting/HistoryEntry.java index 9cd6cef53..54f87f9ea 100644 --- a/core/src/main/java/google/registry/model/reporting/HistoryEntry.java +++ b/core/src/main/java/google/registry/model/reporting/HistoryEntry.java @@ -48,8 +48,6 @@ import google.registry.model.host.HostBase; import google.registry.model.host.HostHistory; import google.registry.model.host.HostHistory.HostHistoryId; import google.registry.model.host.HostResource; -import google.registry.model.replay.DatastoreEntity; -import google.registry.model.replay.SqlEntity; import google.registry.persistence.VKey; import java.util.Optional; import java.util.Set; @@ -86,8 +84,7 @@ import org.joda.time.DateTime; @Entity @MappedSuperclass @Access(AccessType.FIELD) -public class HistoryEntry extends ImmutableObject - implements Buildable, DatastoreEntity, UnsafeSerializable { +public class HistoryEntry extends ImmutableObject implements Buildable, UnsafeSerializable { /** Represents the type of history entry. */ public enum Type { @@ -405,12 +402,6 @@ public class HistoryEntry extends ImmutableObject return resultEntity; } - // In SQL, save the child type - @Override - public Optional toSqlEntity() { - return Optional.of((SqlEntity) toChildHistoryEntity()); - } - /** Creates a {@link VKey} instance from a {@link Key} instance. */ public static VKey createVKey(Key key) { String repoId = key.getParent().getName(); diff --git a/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java b/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java index aec00a2db..5232df8cc 100644 --- a/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java +++ b/core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java @@ -21,7 +21,6 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import com.google.common.collect.ImmutableSet; import google.registry.model.Buildable; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.util.DomainNameUtils; import java.util.Set; import javax.persistence.Column; @@ -40,7 +39,7 @@ import org.joda.time.LocalDate; @Index(name = "spec11threatmatch_tld_idx", columnList = "tld"), @Index(name = "spec11threatmatch_check_date_idx", columnList = "checkDate") }) -public class Spec11ThreatMatch extends ImmutableObject implements Buildable, SqlOnlyEntity { +public class Spec11ThreatMatch extends ImmutableObject implements Buildable { /** The type of threat detected. */ public enum ThreatType { diff --git a/core/src/main/java/google/registry/model/server/Lock.java b/core/src/main/java/google/registry/model/server/Lock.java index d908d3848..fedd47a0d 100644 --- a/core/src/main/java/google/registry/model/server/Lock.java +++ b/core/src/main/java/google/registry/model/server/Lock.java @@ -30,7 +30,6 @@ import com.googlecode.objectify.annotation.Id; import google.registry.model.ImmutableObject; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.persistence.VKey; import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.persistence.transaction.TransactionManager; @@ -62,7 +61,7 @@ import org.joda.time.Duration; @javax.persistence.Entity @Table @IdClass(Lock.LockId.class) -public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Serializable { +public class Lock extends ImmutableObject implements Serializable { private static final long serialVersionUID = 756397280691684645L; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -293,7 +292,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri create(resourceName, scope, requestStatusChecker.getLogId(), now, leaseLength); // Locks are not parented under an EntityGroupRoot (so as to avoid write // contention) and don't need to be backed up. - transactionManager.putIgnoringReadOnlyWithoutBackup(newLock); + transactionManager.put(newLock); return AcquireResult.create(now, lock, newLock, lockState); }; @@ -325,7 +324,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri // Use deleteIgnoringReadOnly() so that we don't create a commit log entry for deleting // the lock. logger.atInfo().log("Deleting lock: %s", lockId); - transactionManager.deleteIgnoringReadOnlyWithoutBackup(key); + transactionManager.delete(key); lockMetrics.recordRelease( resourceName, diff --git a/core/src/main/java/google/registry/model/server/ServerSecret.java b/core/src/main/java/google/registry/model/server/ServerSecret.java index 91079967f..ef965e0f5 100644 --- a/core/src/main/java/google/registry/model/server/ServerSecret.java +++ b/core/src/main/java/google/registry/model/server/ServerSecret.java @@ -27,7 +27,6 @@ import google.registry.model.CacheUtils; import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp.Reason; import google.registry.model.common.CrossTldSingleton; -import google.registry.model.replay.NonReplicatedEntity; import java.nio.ByteBuffer; import java.util.Optional; import java.util.UUID; @@ -41,7 +40,7 @@ import javax.persistence.Transient; @Unindex @NotBackedUp(reason = Reason.AUTO_GENERATED) // TODO(b/27427316): Replace this with an entry in KMSKeyring -public class ServerSecret extends CrossTldSingleton implements NonReplicatedEntity { +public class ServerSecret extends CrossTldSingleton { /** * Cache of the singleton ServerSecret instance that creates it if not present. diff --git a/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java b/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java index c498119fb..cf5867d65 100644 --- a/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java +++ b/core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java @@ -21,7 +21,6 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import java.util.Map; import javax.persistence.CollectionTable; import javax.persistence.Column; @@ -46,7 +45,7 @@ import org.joda.time.DateTime; * functional specifications - SMD Revocation List */ @Entity -public class SignedMarkRevocationList extends ImmutableObject implements SqlOnlyEntity { +public class SignedMarkRevocationList extends ImmutableObject { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/model/tld/Registry.java b/core/src/main/java/google/registry/model/tld/Registry.java index 2d05e3cd3..18435dc80 100644 --- a/core/src/main/java/google/registry/model/tld/Registry.java +++ b/core/src/main/java/google/registry/model/tld/Registry.java @@ -61,7 +61,6 @@ import google.registry.model.common.TimedTransitionProperty; import google.registry.model.common.TimedTransitionProperty.TimedTransition; import google.registry.model.domain.fee.BaseFee.FeeType; import google.registry.model.domain.fee.Fee; -import google.registry.model.replay.DatastoreAndSqlEntity; import google.registry.model.tld.label.PremiumList; import google.registry.model.tld.label.ReservedList; import google.registry.persistence.VKey; @@ -90,8 +89,7 @@ import org.joda.time.Duration; @Entity @javax.persistence.Entity(name = "Tld") @InCrossTld -public class Registry extends ImmutableObject - implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable { +public class Registry extends ImmutableObject implements Buildable, UnsafeSerializable { @Parent @Transient Key parent = getCrossTldKey(); diff --git a/core/src/main/java/google/registry/model/tld/label/PremiumList.java b/core/src/main/java/google/registry/model/tld/label/PremiumList.java index 4c0487fad..5abb1539d 100644 --- a/core/src/main/java/google/registry/model/tld/label/PremiumList.java +++ b/core/src/main/java/google/registry/model/tld/label/PremiumList.java @@ -25,7 +25,6 @@ import com.google.common.hash.BloomFilter; import google.registry.model.Buildable; import google.registry.model.ImmutableObject; import google.registry.model.annotations.ReportedOn; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.tld.Registry; import google.registry.model.tld.label.PremiumList.PremiumEntry; import java.io.Serializable; @@ -53,8 +52,7 @@ import org.joda.money.Money; @ReportedOn @javax.persistence.Entity @Table(indexes = {@Index(columnList = "name", name = "premiumlist_name_idx")}) -public final class PremiumList extends BaseDomainLabelList - implements SqlOnlyEntity { +public final class PremiumList extends BaseDomainLabelList { @Column(nullable = false) CurrencyUnit currency; @@ -120,7 +118,7 @@ public final class PremiumList extends BaseDomainLabelList - implements Buildable, SqlOnlyEntity, Serializable { + implements Buildable, Serializable { @ImmutableObject.Insignificant @javax.persistence.Id Long revisionId; diff --git a/core/src/main/java/google/registry/model/tld/label/ReservedList.java b/core/src/main/java/google/registry/model/tld/label/ReservedList.java index 7d7d0e9ed..75f1ee129 100644 --- a/core/src/main/java/google/registry/model/tld/label/ReservedList.java +++ b/core/src/main/java/google/registry/model/tld/label/ReservedList.java @@ -33,7 +33,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.UncheckedExecutionException; import google.registry.model.Buildable; import google.registry.model.CacheUtils; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.tld.Registry; import google.registry.model.tld.label.DomainLabelMetrics.MetricsReservedListMatch; import java.io.Serializable; @@ -61,8 +60,7 @@ import org.joda.time.DateTime; @javax.persistence.Entity @Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")}) public final class ReservedList - extends BaseDomainLabelList - implements SqlOnlyEntity { + extends BaseDomainLabelList { /** * Mapping from domain name to its reserved list info. @@ -88,7 +86,7 @@ public final class ReservedList /** * Hibernate hook called on the insert of a new ReservedList. Stores the associated {@link - * ReservedEntry}'s. + * ReservedListEntry}'s. * *

We need to persist the list entries, but only on the initial insert (not on update) since * the entries themselves never get changed, so we only annotate it with {@link PostPersist}, not @@ -113,7 +111,7 @@ public final class ReservedList */ @javax.persistence.Entity(name = "ReservedEntry") public static class ReservedListEntry extends DomainLabelEntry - implements Buildable, SqlOnlyEntity, Serializable { + implements Buildable, Serializable { @Insignificant @Id Long revisionId; diff --git a/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java b/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java index cc45933a0..fa407fb16 100644 --- a/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java +++ b/core/src/main/java/google/registry/model/tmch/ClaimsEntry.java @@ -15,7 +15,6 @@ package google.registry.model.tmch; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import java.io.Serializable; import javax.persistence.Column; import javax.persistence.Entity; @@ -30,7 +29,7 @@ import javax.persistence.Id; * work. */ @Entity(name = "ClaimsEntry") -class ClaimsEntry extends ImmutableObject implements SqlOnlyEntity, Serializable { +class ClaimsEntry extends ImmutableObject implements Serializable { @Id private Long revisionId; @Id private String domainLabel; diff --git a/core/src/main/java/google/registry/model/tmch/ClaimsList.java b/core/src/main/java/google/registry/model/tmch/ClaimsList.java index ba9576dae..ec688a5b6 100644 --- a/core/src/main/java/google/registry/model/tmch/ClaimsList.java +++ b/core/src/main/java/google/registry/model/tmch/ClaimsList.java @@ -23,7 +23,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import com.google.common.collect.ImmutableMap; import google.registry.model.CreateAutoTimestamp; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.model.tld.label.ReservedList.ReservedListEntry; import java.util.Map; import java.util.Optional; @@ -52,7 +51,7 @@ import org.joda.time.DateTime; */ @Entity(name = "ClaimsList") @Table -public class ClaimsList extends ImmutableObject implements SqlOnlyEntity { +public class ClaimsList extends ImmutableObject { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/model/tmch/TmchCrl.java b/core/src/main/java/google/registry/model/tmch/TmchCrl.java index dae660667..5f2fb6b16 100644 --- a/core/src/main/java/google/registry/model/tmch/TmchCrl.java +++ b/core/src/main/java/google/registry/model/tmch/TmchCrl.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkNotNull; import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import google.registry.model.common.CrossTldSingleton; -import google.registry.model.replay.SqlOnlyEntity; import java.util.Optional; import javax.annotation.concurrent.Immutable; import javax.persistence.Column; @@ -27,7 +26,7 @@ import org.joda.time.DateTime; /** Singleton for ICANN's TMCH CA certificate revocation list (CRL). */ @javax.persistence.Entity @Immutable -public final class TmchCrl extends CrossTldSingleton implements SqlOnlyEntity { +public final class TmchCrl extends CrossTldSingleton { @Column(name = "certificateRevocations", nullable = false) String crl; diff --git a/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java b/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java index 1ab51d66d..9696976af 100644 --- a/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java +++ b/core/src/main/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactory.java @@ -25,7 +25,6 @@ import com.google.common.collect.Ordering; import com.googlecode.objectify.Key; import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.ofy.CommitLogManifest; -import google.registry.persistence.transaction.Transaction; import org.joda.time.DateTime; /** @@ -69,12 +68,6 @@ public final class CommitLogRevisionsTranslatorFactory @Override ImmutableSortedMap> transformBeforeSave( ImmutableSortedMap> revisions) { - - // Don't do anything if we're just doing object serialization. - if (Transaction.inSerializationMode()) { - return revisions; - } - DateTime now = ofyTm().getTransactionTime(); DateTime threshold = now.minus(getCommitLogDatastoreRetention()); DateTime preThresholdTime = firstNonNull(revisions.floorKey(threshold), START_OF_TIME); diff --git a/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java b/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java index 94d29c634..6bd1636af 100644 --- a/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java +++ b/core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java @@ -14,13 +14,10 @@ package google.registry.model.translators; -import static com.google.common.base.Preconditions.checkState; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; import static org.joda.time.DateTimeZone.UTC; import google.registry.model.UpdateAutoTimestamp; -import google.registry.persistence.transaction.Transaction; import java.util.Date; import org.joda.time.DateTime; @@ -49,19 +46,6 @@ public class UpdateAutoTimestampTranslatorFactory /** Save a timestamp, setting it to the current time. */ @Override public Date saveValue(UpdateAutoTimestamp pojoValue) { - - // If we're in the course of Transaction serialization, we have to use the transaction time - // here and the JPA transaction manager which is what will ultimately be saved during the - // commit. - // Note that this branch doesn't respect "auto update disabled", as this state is - // specifically to address replay, so we add a runtime check for this. - if (Transaction.inSerializationMode()) { - checkState( - UpdateAutoTimestamp.autoUpdateEnabled(), - "Auto-update disabled during transaction serialization."); - return jpaTm().getTransactionTime().toDate(); - } - return UpdateAutoTimestamp.autoUpdateEnabled() ? ofyTm().getTransactionTime().toDate() : pojoValue.getTimestamp().toDate(); diff --git a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java index adebb252f..a34960903 100644 --- a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java @@ -25,7 +25,6 @@ import google.registry.batch.DeleteExpiredDomainsAction; import google.registry.batch.DeleteLoadTestDataAction; import google.registry.batch.DeleteProberDataAction; import google.registry.batch.ExpandRecurringBillingEventsAction; -import google.registry.batch.RefreshDnsOnHostRenameAction; import google.registry.batch.RelockDomainAction; import google.registry.batch.ResaveAllEppResourcesAction; import google.registry.batch.ResaveAllEppResourcesPipelineAction; @@ -180,8 +179,6 @@ interface BackendRequestComponent { RefreshDnsAction refreshDnsAction(); - RefreshDnsOnHostRenameAction refreshDnsOnHostRenameAction(); - RelockDomainAction relockDomainAction(); ResaveAllEppResourcesAction resaveAllEppResourcesAction(); diff --git a/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java b/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java index 5558628c4..5b4a0677e 100644 --- a/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java +++ b/core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java @@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static google.registry.model.ofy.DatastoreTransactionManager.toSqlEntity; import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import static java.util.AbstractMap.SimpleEntry; import static java.util.stream.Collectors.joining; @@ -30,17 +29,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.flogger.FluentLogger; -import com.googlecode.objectify.Key; import google.registry.model.ImmutableObject; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection; import google.registry.model.index.EppResourceIndex; import google.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex; import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex; import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex; -import google.registry.model.ofy.DatastoreTransactionManager; -import google.registry.model.replay.NonReplicatedEntity; -import google.registry.model.replay.SqlOnlyEntity; import google.registry.persistence.JpaRetries; import google.registry.persistence.VKey; import google.registry.util.Clock; @@ -137,15 +130,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { // Postgresql-specific: 'set transaction' command must be called inside a transaction assertInTransaction(); - ReadOnlyCheckingEntityManager entityManager = - (ReadOnlyCheckingEntityManager) getEntityManager(); + EntityManager entityManager = getEntityManager(); // Isolation is hardcoded to REPEATABLE READ, as specified by parent's Javadoc. entityManager .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - .executeUpdateIgnoringReadOnly(); + .executeUpdate(); entityManager .createNativeQuery(String.format("SET TRANSACTION SNAPSHOT '%s'", snapshotId)) - .executeUpdateIgnoringReadOnly(); + .executeUpdate(); return this; } @@ -178,45 +170,12 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { @Override public T transact(Supplier work) { - return transact(work, true); + return retrier.callWithRetry(() -> transactNoRetry(work), JpaRetries::isFailedTxnRetriable); } @Override public T transactWithoutBackup(Supplier work) { - return transact(work, false); - } - - private T transact(Supplier work, boolean withBackup) { - return retrier.callWithRetry( - () -> { - if (inTransaction()) { - return work.get(); - } - TransactionInfo txnInfo = transactionInfo.get(); - txnInfo.entityManager = createReadOnlyCheckingEntityManager(); - EntityTransaction txn = txnInfo.entityManager.getTransaction(); - try { - txn.begin(); - txnInfo.start(clock, withBackup); - T result = work.get(); - txnInfo.recordTransaction(); - txn.commit(); - return result; - } catch (RuntimeException | Error e) { - // Error is unchecked! - try { - txn.rollback(); - logger.atWarning().log("Error during transaction; transaction rolled back."); - } catch (Throwable rollbackException) { - logger.atSevere().withCause(rollbackException).log( - "Rollback failed; suppressing error."); - } - throw e; - } finally { - txnInfo.clear(); - } - }, - JpaRetries::isFailedTxnRetriable); + return transact(work); } @Override @@ -225,13 +184,12 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return work.get(); } TransactionInfo txnInfo = transactionInfo.get(); - txnInfo.entityManager = createReadOnlyCheckingEntityManager(); + txnInfo.entityManager = emf.createEntityManager(); EntityTransaction txn = txnInfo.entityManager.getTransaction(); try { txn.begin(); - txnInfo.start(clock, true); + txnInfo.start(clock); T result = work.get(); - txnInfo.recordTransaction(); txn.commit(); return result; } catch (RuntimeException | Error e) { @@ -320,9 +278,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return; } assertInTransaction(); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - transactionInfo.get().insertObject(toPersist); + transactionInfo.get().insertObject(entity); } @Override @@ -354,9 +310,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return; } assertInTransaction(); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - transactionInfo.get().updateObject(toPersist); + transactionInfo.get().updateObject(entity); } @Override @@ -393,9 +347,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { } assertInTransaction(); checkArgument(exists(entity), "Given entity does not exist"); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - transactionInfo.get().updateObject(toPersist); + transactionInfo.get().updateObject(entity); } @Override @@ -431,7 +383,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { @Override public boolean exists(Object entity) { checkArgumentNotNull(entity, "entity must be specified"); - entity = toSqlEntity(entity); EntityType entityType = getEntityType(entity.getClass()); ImmutableSet entityIds = getEntityIdsFromEntity(entityType, entity); return exists(entityType.getName(), entityIds); @@ -475,7 +426,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { @Override public ImmutableList loadByEntitiesIfPresent(Iterable entities) { return Streams.stream(entities) - .map(DatastoreTransactionManager::toSqlEntity) .filter(this::exists) .map(this::loadByEntity) .collect(toImmutableList()); @@ -510,16 +460,14 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { public T loadByEntity(T entity) { checkArgumentNotNull(entity, "entity must be specified"); assertInTransaction(); - // If the caller requested a HistoryEntry, load the corresponding *History class - T possibleChild = toSqlEntity(entity); @SuppressWarnings("unchecked") T returnValue = (T) loadByKey( VKey.createSql( - possibleChild.getClass(), + entity.getClass(), // Casting to Serializable is safe according to JPA (JSR 338 sec. 2.4). - (Serializable) emf.getPersistenceUnitUtil().getIdentifier(possibleChild))); + (Serializable) emf.getPersistenceUnitUtil().getIdentifier(entity))); return returnValue; } @@ -570,7 +518,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { String.format("DELETE FROM %s WHERE %s", entityType.getName(), getAndClause(entityIds)); Query query = query(sql); entityIds.forEach(entityId -> query.setParameter(entityId.name, entityId.value)); - transactionInfo.get().addDelete(key); return query.executeUpdate(); } @@ -592,7 +539,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return entity; } assertInTransaction(); - entity = toSqlEntity(entity); T managedEntity = entity; if (!getEntityManager().contains(entity)) { // We don't add the entity to "objectsToSave": once deleted, the object should never be @@ -600,13 +546,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { managedEntity = getEntityManager().merge(entity); } getEntityManager().remove(managedEntity); - - // We check shouldReplicate() in TransactionInfo.addDelete(), but we have to check it here as - // well prior to attempting to create a datastore key because a non-replicated entity may not - // have one. - if (shouldReplicate(entity.getClass())) { - transactionInfo.get().addDelete(VKey.from(Key.create(entity))); - } return managedEntity; } @@ -640,38 +579,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return false; } - @Override - public void putIgnoringReadOnlyWithoutBackup(Object entity) { - checkArgumentNotNull(entity); - if (isEntityOfIgnoredClass(entity)) { - return; - } - assertInTransaction(); - // Necessary due to the changes in HistoryEntry representation during the migration to SQL - Object toPersist = toSqlEntity(entity); - TransactionInfo txn = transactionInfo.get(); - Object merged = txn.entityManager.mergeIgnoringReadOnly(toPersist); - txn.objectsToSave.add(merged); - txn.addUpdate(toPersist); - } - - @Override - public void deleteIgnoringReadOnlyWithoutBackup(VKey key) { - checkArgumentNotNull(key, "key must be specified"); - assertInTransaction(); - if (IGNORED_ENTITY_CLASSES.contains(key.getKind())) { - return; - } - EntityType entityType = getEntityType(key.getKind()); - ImmutableSet entityIds = getEntityIdsFromSqlKey(entityType, key.getSqlKey()); - String sql = - String.format("DELETE FROM %s WHERE %s", entityType.getName(), getAndClause(entityIds)); - ReadOnlyCheckingQuery query = transactionInfo.get().entityManager.createQuery(sql); - entityIds.forEach(entityId -> query.setParameter(entityId.name, entityId.value)); - transactionInfo.get().addDelete(key); - query.executeUpdateIgnoringReadOnly(); - } - @Override public void assertDelete(VKey key) { if (internalDelete(key) != 1) { @@ -680,10 +587,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { } } - private ReadOnlyCheckingEntityManager createReadOnlyCheckingEntityManager() { - return new ReadOnlyCheckingEntityManager(emf.createEntityManager()); - } - private EntityType getEntityType(Class clazz) { return emf.getMetamodel().entity(clazz); } @@ -820,46 +723,28 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { return entity; } - /** Returns true if the entity class should be replicated from SQL to datastore. */ - private static boolean shouldReplicate(Class entityClass) { - return !NonReplicatedEntity.class.isAssignableFrom(entityClass) - && !SqlOnlyEntity.class.isAssignableFrom(entityClass); - } - private static class TransactionInfo { - ReadOnlyCheckingEntityManager entityManager; + EntityManager entityManager; boolean inTransaction = false; DateTime transactionTime; - // Serializable representation of the transaction to be persisted in the Transaction table. - Transaction.Builder contentsBuilder; - // The set of entity objects that have been either persisted (via insert()) or merged (via // put()/update()). If the entity manager returns these as a result of a find() or query // operation, we can not detach them -- detaching removes them from the transaction and causes // them to not be saved to the database -- so we throw an exception instead. - Set objectsToSave = Collections.newSetFromMap(new IdentityHashMap()); + Set objectsToSave = Collections.newSetFromMap(new IdentityHashMap<>()); /** Start a new transaction. */ - private void start(Clock clock, boolean withBackup) { + private void start(Clock clock) { checkArgumentNotNull(clock); inTransaction = true; transactionTime = clock.nowUtc(); - Supplier sqlToDsReplaySupplier = - () -> - DatabaseMigrationStateSchedule.getValueAtTime(transactionTime) - .getReplayDirection() - .equals(ReplayDirection.SQL_TO_DATASTORE); - if (withBackup && sqlToDsReplaySupplier.get()) { - contentsBuilder = new Transaction.Builder(); - } } private void clear() { inTransaction = false; transactionTime = null; - contentsBuilder = null; - objectsToSave = Collections.newSetFromMap(new IdentityHashMap()); + objectsToSave = Collections.newSetFromMap(new IdentityHashMap<>()); if (entityManager != null) { // Close this EntityManager just let the connection pool be able to reuse it, it doesn't // close the underlying database connection. @@ -868,39 +753,16 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager { } } - private void addUpdate(Object entity) { - if (contentsBuilder != null && shouldReplicate(entity.getClass())) { - contentsBuilder.addUpdate(entity); - } - } - - private void addDelete(VKey key) { - if (contentsBuilder != null && shouldReplicate(key.getKind())) { - contentsBuilder.addDelete(key); - } - } - - private void recordTransaction() { - if (contentsBuilder != null) { - Transaction persistedTxn = contentsBuilder.build(); - if (!persistedTxn.isEmpty()) { - entityManager.persist(persistedTxn.toEntity()); - } - } - } - /** Does the full "update" on an object including all internal housekeeping. */ private void updateObject(Object object) { Object merged = entityManager.merge(object); objectsToSave.add(merged); - addUpdate(object); } /** Does the full "insert" on a new object including all internal housekeeping. */ private void insertObject(Object object) { entityManager.persist(object); objectsToSave.add(object); - addUpdate(object); } /** Returns true if the object has been persisted/merged and will be saved on commit. */ diff --git a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java b/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java deleted file mode 100644 index d2e929bee..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java +++ /dev/null @@ -1,322 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.List; -import java.util.Map; -import javax.persistence.EntityGraph; -import javax.persistence.EntityManager; -import javax.persistence.EntityManagerFactory; -import javax.persistence.EntityTransaction; -import javax.persistence.FlushModeType; -import javax.persistence.LockModeType; -import javax.persistence.Query; -import javax.persistence.StoredProcedureQuery; -import javax.persistence.TypedQuery; -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaDelete; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.CriteriaUpdate; -import javax.persistence.metamodel.Metamodel; - -/** An {@link EntityManager} that throws exceptions on write actions if in read-only mode. */ -@DeleteAfterMigration -public class ReadOnlyCheckingEntityManager implements EntityManager { - - private final EntityManager delegate; - - public ReadOnlyCheckingEntityManager(EntityManager delegate) { - this.delegate = delegate; - } - - @Override - public void persist(Object entity) { - assertNotReadOnlyMode(); - delegate.persist(entity); - } - - @Override - public T merge(T entity) { - assertNotReadOnlyMode(); - return delegate.merge(entity); - } - - @Override - public void remove(Object entity) { - assertNotReadOnlyMode(); - delegate.remove(entity); - } - - @Override - public T find(Class entityClass, Object primaryKey) { - return delegate.find(entityClass, primaryKey); - } - - @Override - public T find(Class entityClass, Object primaryKey, Map properties) { - return delegate.find(entityClass, primaryKey, properties); - } - - @Override - public T find(Class entityClass, Object primaryKey, LockModeType lockMode) { - return delegate.find(entityClass, primaryKey, lockMode); - } - - @Override - public T find( - Class entityClass, - Object primaryKey, - LockModeType lockMode, - Map properties) { - return delegate.find(entityClass, primaryKey, lockMode, properties); - } - - @Override - public T getReference(Class entityClass, Object primaryKey) { - return delegate.getReference(entityClass, primaryKey); - } - - @Override - public void flush() { - delegate.flush(); - } - - @Override - public void setFlushMode(FlushModeType flushMode) { - delegate.setFlushMode(flushMode); - } - - @Override - public FlushModeType getFlushMode() { - return delegate.getFlushMode(); - } - - @Override - public void lock(Object entity, LockModeType lockMode) { - assertNotReadOnlyMode(); - delegate.lock(entity, lockMode); - } - - @Override - public void lock(Object entity, LockModeType lockMode, Map properties) { - assertNotReadOnlyMode(); - delegate.lock(entity, lockMode, properties); - } - - @Override - public void refresh(Object entity) { - delegate.refresh(entity); - } - - @Override - public void refresh(Object entity, Map properties) { - delegate.refresh(entity, properties); - } - - @Override - public void refresh(Object entity, LockModeType lockMode) { - delegate.refresh(entity, lockMode); - } - - @Override - public void refresh(Object entity, LockModeType lockMode, Map properties) { - delegate.refresh(entity, lockMode, properties); - } - - @Override - public void clear() { - delegate.clear(); - } - - @Override - public void detach(Object entity) { - delegate.detach(entity); - } - - @Override - public boolean contains(Object entity) { - return delegate.contains(entity); - } - - @Override - public LockModeType getLockMode(Object entity) { - return delegate.getLockMode(entity); - } - - @Override - public void setProperty(String propertyName, Object value) { - delegate.setProperty(propertyName, value); - } - - @Override - public Map getProperties() { - return delegate.getProperties(); - } - - @Override - public ReadOnlyCheckingQuery createQuery(String qlString) { - return new ReadOnlyCheckingQuery(delegate.createQuery(qlString)); - } - - @Override - public TypedQuery createQuery(CriteriaQuery criteriaQuery) { - return new ReadOnlyCheckingTypedQuery<>(delegate.createQuery(criteriaQuery)); - } - - @Override - public Query createQuery(CriteriaUpdate updateQuery) { - assertNotReadOnlyMode(); - return delegate.createQuery(updateQuery); - } - - @Override - public Query createQuery(CriteriaDelete deleteQuery) { - assertNotReadOnlyMode(); - return delegate.createQuery(deleteQuery); - } - - @Override - public TypedQuery createQuery(String qlString, Class resultClass) { - return new ReadOnlyCheckingTypedQuery<>(delegate.createQuery(qlString, resultClass)); - } - - @Override - public Query createNamedQuery(String name) { - return new ReadOnlyCheckingQuery(delegate.createNamedQuery(name)); - } - - @Override - public TypedQuery createNamedQuery(String name, Class resultClass) { - return new ReadOnlyCheckingTypedQuery<>(delegate.createNamedQuery(name, resultClass)); - } - - @Override - public ReadOnlyCheckingQuery createNativeQuery(String sqlString) { - return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString)); - } - - @Override - public Query createNativeQuery(String sqlString, Class resultClass) { - return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString, resultClass)); - } - - @Override - public Query createNativeQuery(String sqlString, String resultSetMapping) { - return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString, resultSetMapping)); - } - - @Override - public StoredProcedureQuery createNamedStoredProcedureQuery(String name) { - assertNotReadOnlyMode(); - return delegate.createNamedStoredProcedureQuery(name); - } - - @Override - public StoredProcedureQuery createStoredProcedureQuery(String procedureName) { - assertNotReadOnlyMode(); - return delegate.createStoredProcedureQuery(procedureName); - } - - @Override - public StoredProcedureQuery createStoredProcedureQuery( - String procedureName, Class... resultClasses) { - assertNotReadOnlyMode(); - return delegate.createStoredProcedureQuery(procedureName, resultClasses); - } - - @Override - public StoredProcedureQuery createStoredProcedureQuery( - String procedureName, String... resultSetMappings) { - assertNotReadOnlyMode(); - return delegate.createStoredProcedureQuery(procedureName, resultSetMappings); - } - - @Override - public void joinTransaction() { - delegate.joinTransaction(); - } - - @Override - public boolean isJoinedToTransaction() { - return delegate.isJoinedToTransaction(); - } - - @Override - public T unwrap(Class cls) { - return delegate.unwrap(cls); - } - - @Override - public Object getDelegate() { - return delegate.getDelegate(); - } - - @Override - public void close() { - delegate.close(); - } - - @Override - public boolean isOpen() { - return delegate.isOpen(); - } - - @Override - public EntityTransaction getTransaction() { - return delegate.getTransaction(); - } - - @Override - public EntityManagerFactory getEntityManagerFactory() { - return delegate.getEntityManagerFactory(); - } - - @Override - public CriteriaBuilder getCriteriaBuilder() { - return delegate.getCriteriaBuilder(); - } - - @Override - public Metamodel getMetamodel() { - return delegate.getMetamodel(); - } - - @Override - public EntityGraph createEntityGraph(Class rootType) { - return delegate.createEntityGraph(rootType); - } - - @Override - public EntityGraph createEntityGraph(String graphName) { - return delegate.createEntityGraph(graphName); - } - - @Override - public EntityGraph getEntityGraph(String graphName) { - return delegate.getEntityGraph(graphName); - } - - @Override - public List> getEntityGraphs(Class entityClass) { - return delegate.getEntityGraphs(entityClass); - } - - public T mergeIgnoringReadOnly(T entity) { - return delegate.merge(entity); - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingQuery.java b/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingQuery.java deleted file mode 100644 index 345946253..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingQuery.java +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; -import javax.persistence.FlushModeType; -import javax.persistence.LockModeType; -import javax.persistence.Parameter; -import javax.persistence.Query; -import javax.persistence.TemporalType; - -/** A {@link Query} that throws exceptions on write actions if in read-only mode. */ -@DeleteAfterMigration -class ReadOnlyCheckingQuery implements Query { - - private final Query delegate; - - ReadOnlyCheckingQuery(Query delegate) { - this.delegate = delegate; - } - - @Override - public List getResultList() { - return delegate.getResultList(); - } - - @Override - public Stream getResultStream() { - return delegate.getResultStream(); - } - - @Override - public Object getSingleResult() { - return delegate.getSingleResult(); - } - - @Override - public int executeUpdate() { - assertNotReadOnlyMode(); - return delegate.executeUpdate(); - } - - @Override - public Query setMaxResults(int maxResult) { - return delegate.setMaxResults(maxResult); - } - - @Override - public int getMaxResults() { - return delegate.getMaxResults(); - } - - @Override - public Query setFirstResult(int startPosition) { - return delegate.setFirstResult(startPosition); - } - - @Override - public int getFirstResult() { - return delegate.getFirstResult(); - } - - @Override - public Query setHint(String hintName, Object value) { - return delegate.setHint(hintName, value); - } - - @Override - public Map getHints() { - return delegate.getHints(); - } - - @Override - public Query setParameter(Parameter param, T value) { - return delegate.setParameter(param, value); - } - - @Override - public Query setParameter(Parameter param, Calendar value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public Query setParameter(Parameter param, Date value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public Query setParameter(String name, Object value) { - return delegate.setParameter(name, value); - } - - @Override - public Query setParameter(String name, Calendar value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public Query setParameter(String name, Date value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public Query setParameter(int position, Object value) { - return delegate.setParameter(position, value); - } - - @Override - public Query setParameter(int position, Calendar value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public Query setParameter(int position, Date value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public Set> getParameters() { - return delegate.getParameters(); - } - - @Override - public Parameter getParameter(String name) { - return delegate.getParameter(name); - } - - @Override - public Parameter getParameter(String name, Class type) { - return delegate.getParameter(name, type); - } - - @Override - public Parameter getParameter(int position) { - return delegate.getParameter(position); - } - - @Override - public Parameter getParameter(int position, Class type) { - return delegate.getParameter(position, type); - } - - @Override - public boolean isBound(Parameter param) { - return delegate.isBound(param); - } - - @Override - public T getParameterValue(Parameter param) { - return delegate.getParameterValue(param); - } - - @Override - public Object getParameterValue(String name) { - return delegate.getParameterValue(name); - } - - @Override - public Object getParameterValue(int position) { - return delegate.getParameterValue(position); - } - - @Override - public Query setFlushMode(FlushModeType flushMode) { - return delegate.setFlushMode(flushMode); - } - - @Override - public FlushModeType getFlushMode() { - return delegate.getFlushMode(); - } - - @Override - public Query setLockMode(LockModeType lockMode) { - return delegate.setLockMode(lockMode); - } - - @Override - public LockModeType getLockMode() { - return delegate.getLockMode(); - } - - @Override - public T unwrap(Class cls) { - return delegate.unwrap(cls); - } - - public int executeUpdateIgnoringReadOnly() { - return delegate.executeUpdate(); - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingTypedQuery.java b/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingTypedQuery.java deleted file mode 100644 index e3eab9e2f..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingTypedQuery.java +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static google.registry.persistence.transaction.TransactionManagerFactory.assertNotReadOnlyMode; - -import google.registry.model.annotations.DeleteAfterMigration; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; -import javax.persistence.FlushModeType; -import javax.persistence.LockModeType; -import javax.persistence.Parameter; -import javax.persistence.TemporalType; -import javax.persistence.TypedQuery; - -/** A {@link TypedQuery } that throws exceptions on write actions if in read-only mode. */ -@DeleteAfterMigration -class ReadOnlyCheckingTypedQuery implements TypedQuery { - - private final TypedQuery delegate; - - ReadOnlyCheckingTypedQuery(TypedQuery delegate) { - this.delegate = delegate; - } - - @Override - public List getResultList() { - return delegate.getResultList(); - } - - @Override - public Stream getResultStream() { - return delegate.getResultStream(); - } - - @Override - public T getSingleResult() { - return delegate.getSingleResult(); - } - - @Override - public int executeUpdate() { - assertNotReadOnlyMode(); - return delegate.executeUpdate(); - } - - @Override - public TypedQuery setMaxResults(int maxResult) { - return delegate.setMaxResults(maxResult); - } - - @Override - public int getMaxResults() { - return delegate.getMaxResults(); - } - - @Override - public TypedQuery setFirstResult(int startPosition) { - return delegate.setFirstResult(startPosition); - } - - @Override - public int getFirstResult() { - return delegate.getFirstResult(); - } - - @Override - public TypedQuery setHint(String hintName, Object value) { - return delegate.setHint(hintName, value); - } - - @Override - public Map getHints() { - return delegate.getHints(); - } - - @Override - public TypedQuery setParameter(Parameter param, T1 value) { - return delegate.setParameter(param, value); - } - - @Override - public TypedQuery setParameter( - Parameter param, Calendar value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public TypedQuery setParameter(Parameter param, Date value, TemporalType temporalType) { - return delegate.setParameter(param, value, temporalType); - } - - @Override - public TypedQuery setParameter(String name, Object value) { - return delegate.setParameter(name, value); - } - - @Override - public TypedQuery setParameter(String name, Calendar value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public TypedQuery setParameter(String name, Date value, TemporalType temporalType) { - return delegate.setParameter(name, value, temporalType); - } - - @Override - public TypedQuery setParameter(int position, Object value) { - return delegate.setParameter(position, value); - } - - @Override - public TypedQuery setParameter(int position, Calendar value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public TypedQuery setParameter(int position, Date value, TemporalType temporalType) { - return delegate.setParameter(position, value, temporalType); - } - - @Override - public Set> getParameters() { - return delegate.getParameters(); - } - - @Override - public Parameter getParameter(String name) { - return delegate.getParameter(name); - } - - @Override - public Parameter getParameter(String name, Class type) { - return delegate.getParameter(name, type); - } - - @Override - public Parameter getParameter(int position) { - return delegate.getParameter(position); - } - - @Override - public Parameter getParameter(int position, Class type) { - return delegate.getParameter(position, type); - } - - @Override - public boolean isBound(Parameter param) { - return delegate.isBound(param); - } - - @Override - public X getParameterValue(Parameter param) { - return delegate.getParameterValue(param); - } - - @Override - public Object getParameterValue(String name) { - return delegate.getParameterValue(name); - } - - @Override - public Object getParameterValue(int position) { - return delegate.getParameterValue(position); - } - - @Override - public TypedQuery setFlushMode(FlushModeType flushMode) { - return delegate.setFlushMode(flushMode); - } - - @Override - public FlushModeType getFlushMode() { - return delegate.getFlushMode(); - } - - @Override - public TypedQuery setLockMode(LockModeType lockMode) { - return delegate.setLockMode(lockMode); - } - - @Override - public LockModeType getLockMode() { - return delegate.getLockMode(); - } - - @Override - public X unwrap(Class cls) { - return delegate.unwrap(cls); - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/Transaction.java b/core/src/main/java/google/registry/persistence/transaction/Transaction.java deleted file mode 100644 index 75d55a36a..000000000 --- a/core/src/main/java/google/registry/persistence/transaction/Transaction.java +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; - -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityTranslator; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; -import google.registry.model.Buildable; -import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlEntity; -import google.registry.persistence.VKey; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.ObjectStreamClass; - -/** - * A SQL transaction that can be serialized and stored in its own table. - * - *

Transaction is used to store transactions committed to Cloud SQL in a Transaction table during - * the second phase of our migration, during which time we will be asynchronously replaying Cloud - * SQL transactions to datastore. - * - *

TODO(mmuller): Use these from {@link TransactionManager} to store the contents of an SQL - * transaction for asynchronous propagation to datastore. Implement a cron endpoint that reads them - * from the Transaction table and calls writeToDatastore(). - */ -public class Transaction extends ImmutableObject implements Buildable { - - // Version id for persisted objects. Use the creation date for the value, as it's reasonably - // unique and inherently informative. - private static final int VERSION_ID = 20200604; - - // Keep a per-thread flag to keep track of whether we're serializing an entity for a transaction. - // This is used by internal translators to avoid doing things that are dependent on being in a - // datastore transaction and alter the persisted representation of the entity. - private static ThreadLocal inSerializationMode = ThreadLocal.withInitial(() -> false); - - private transient ImmutableList mutations; - - @VisibleForTesting - public ImmutableList getMutations() { - return mutations; - } - - /** Write the entire transaction to the datastore in a datastore transaction. */ - public void writeToDatastore() { - ofyTm() - .transact( - () -> { - for (Mutation mutation : mutations) { - mutation.writeToDatastore(); - } - }); - } - - /** Serialize a transaction to a byte array. */ - public byte[] serialize() { - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(baos); - - // Write the transaction version id. This serves as both a version id and a "magic number" to - // protect us against trying to deserialize some random byte array. - out.writeInt(VERSION_ID); - - // Write all of the mutations, preceded by their count. - out.writeInt(mutations.size()); - try { - inSerializationMode.set(true); - for (Mutation mutation : mutations) { - mutation.serializeTo(out); - } - } finally { - inSerializationMode.set(false); - } - - out.close(); - return baos.toByteArray(); - } catch (IOException e) { - throw new IllegalArgumentException(); - } - } - - public static Transaction deserialize(byte[] serializedTransaction) throws IOException { - ObjectInputStream in = - new LenientObjectInputStream(new ByteArrayInputStream(serializedTransaction)); - - // Verify that the data is what we expect. - int version = in.readInt(); - checkArgument( - version == VERSION_ID, "Invalid version id. Expected %s but got %s", VERSION_ID, version); - - Transaction.Builder builder = new Transaction.Builder(); - int mutationCount = in.readInt(); - for (int i = 0; i < mutationCount; ++i) { - try { - builder.add(Mutation.deserializeFrom(in)); - } catch (EOFException e) { - throw new RuntimeException("Serialized transaction terminated prematurely", e); - } - } - if (in.read() != -1) { - throw new RuntimeException("Unread data at the end of a serialized transaction."); - } - return builder.build(); - } - - /** Returns true if the transaction contains no mutations. */ - public boolean isEmpty() { - return mutations.isEmpty(); - } - - /** - * Returns true if we are serializing a transaction in the current thread. - * - *

This should be checked by any Ofy translators prior to making any changes to an entity's - * state representation based on the assumption that we are currently persisting the entity to - * datastore. - */ - public static boolean inSerializationMode() { - return inSerializationMode.get(); - } - - @Override - public Builder asBuilder() { - return new Builder(clone(this)); - } - - public final TransactionEntity toEntity() { - return new TransactionEntity(serialize()); - } - - public static class Builder extends GenericBuilder { - - ImmutableList.Builder listBuilder = new ImmutableList.Builder<>(); - - Builder() {} - - protected Builder(Transaction instance) { - super(instance); - } - - public Builder addUpdate(Object entity) { - checkNotNull(entity); - listBuilder.add(new Update(entity)); - return thisCastToDerived(); - } - - public Builder addDelete(VKey key) { - checkNotNull(key); - listBuilder.add(new Delete(key)); - return thisCastToDerived(); - } - - /** Adds a mutation (mainly intended for serialization) */ - Builder add(Mutation mutation) { - checkNotNull(mutation); - listBuilder.add(mutation); - return thisCastToDerived(); - } - - @Override - public Transaction build() { - getInstance().mutations = listBuilder.build(); - return super.build(); - } - } - - /** Base class for database record mutations. */ - public abstract static class Mutation { - - enum Type { - UPDATE, - DELETE - } - - /** Write the changes in the mutation to the datastore. */ - public abstract void writeToDatastore(); - - /** Serialize the mutation to the output stream. */ - public abstract void serializeTo(ObjectOutputStream out) throws IOException; - - /** Deserialize a mutation from the input stream. */ - public static Mutation deserializeFrom(ObjectInputStream in) throws IOException { - try { - Type type = (Type) in.readObject(); - switch (type) { - case UPDATE: - return Update.deserializeFrom(in); - case DELETE: - return Delete.deserializeFrom(in); - default: - throw new IllegalArgumentException("Unknown enum value: " + type); - } - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - } - - /** - * Record update. - * - *

Note that we don't have to distinguish between add and update, since this is for replay into - * the datastore which makes no such distinction. - * - *

Update serializes its entity using Objectify serialization. - */ - public static class Update extends Mutation { - private Object entity; - - Update(Object entity) { - this.entity = - (entity instanceof SqlEntity) ? ((SqlEntity) entity).toDatastoreEntity().get() : entity; - } - - @Override - public void writeToDatastore() { - ofyTm().putIgnoringReadOnlyWithBackup(entity); - } - - @Override - public void serializeTo(ObjectOutputStream out) throws IOException { - out.writeObject(Type.UPDATE); - Entity realEntity = auditedOfy().toEntity(entity); - EntityProto proto = EntityTranslator.convertToPb(realEntity); - out.write(VERSION_ID); - proto.writeDelimitedTo(out); - } - - @VisibleForTesting - public Object getEntity() { - return entity; - } - - public static Update deserializeFrom(ObjectInputStream in) throws IOException { - EntityProto proto = new EntityProto(); - proto.parseDelimitedFrom(in); - return new Update(auditedOfy().toPojo(EntityTranslator.createFromPb(proto))); - } - } - - /** - * Record deletion. - * - *

Delete serializes its VKey using Java native serialization. - */ - public static class Delete extends Mutation { - private final VKey key; - - Delete(VKey key) { - this.key = key; - } - - @Override - public void writeToDatastore() { - ofyTm().deleteIgnoringReadOnlyWithBackup(key); - } - - @Override - public void serializeTo(ObjectOutputStream out) throws IOException { - out.writeObject(Type.DELETE); - - // Java object serialization works for this. - out.writeObject(key); - } - - @VisibleForTesting - public VKey getKey() { - return key; - } - - public static Delete deserializeFrom(ObjectInputStream in) throws IOException { - try { - return new Delete((VKey) in.readObject()); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } - } - } - - /** - * ObjectInputStream that ignores the UIDs of serialized objects. - * - *

We only really need to deserialize VKeys. However, VKeys have a class object associated with - * them, and if the class is changed and we haven't defined a serialVersionUID for it, we get an - * exception during deserialization. - * - *

It's safe for us to ignore this condition: we only care about attaching the correct local - * class object to the VKey. So this class effectively does so by replacing the class descriptor - * if it's version UID doesn't match that of the local class. - */ - private static class LenientObjectInputStream extends ObjectInputStream { - - public LenientObjectInputStream(InputStream in) throws IOException { - super(in); - } - - @Override - protected ObjectStreamClass readClassDescriptor() throws IOException, ClassNotFoundException { - ObjectStreamClass persistedDescriptor = super.readClassDescriptor(); - Class localClass = Class.forName(persistedDescriptor.getName()); - ObjectStreamClass localDescriptor = ObjectStreamClass.lookup(localClass); - if (localDescriptor != null) { - if (persistedDescriptor.getSerialVersionUID() != localDescriptor.getSerialVersionUID()) { - return localDescriptor; - } - } - return persistedDescriptor; - } - } -} diff --git a/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java b/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java index af6db5077..4b9022544 100644 --- a/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java +++ b/core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java @@ -16,7 +16,7 @@ package google.registry.persistence.transaction; import com.google.common.annotations.VisibleForTesting; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlOnlyEntity; +import google.registry.model.annotations.DeleteAfterMigration; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; @@ -30,7 +30,8 @@ import javax.persistence.Table; */ @Entity @Table(name = "Transaction") -public class TransactionEntity extends ImmutableObject implements SqlOnlyEntity { +@DeleteAfterMigration +public class TransactionEntity extends ImmutableObject { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) diff --git a/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java b/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java index b26cadf9d..2cdcc5ed8 100644 --- a/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java +++ b/core/src/main/java/google/registry/persistence/transaction/TransactionManager.java @@ -311,10 +311,4 @@ public interface TransactionManager { /** Returns true if the transaction manager is DatastoreTransactionManager, false otherwise. */ boolean isOfy(); - - /** Performs the write ignoring any read-only restrictions or backup, for use only in replay. */ - void putIgnoringReadOnlyWithoutBackup(Object entity); - - /** Performs the delete ignoring any read-only restrictions or backup, for use only in replay. */ - void deleteIgnoringReadOnlyWithoutBackup(VKey key); } diff --git a/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java b/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java index 215e6bf67..5f643ecc2 100644 --- a/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java +++ b/core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java @@ -15,7 +15,6 @@ package google.registry.persistence.transaction; import static com.google.common.base.Preconditions.checkState; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC; import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; import com.google.appengine.api.utils.SystemProperty; @@ -23,15 +22,10 @@ import com.google.appengine.api.utils.SystemProperty.Environment.Value; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Suppliers; import google.registry.config.RegistryEnvironment; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.model.ofy.DatastoreTransactionManager; import google.registry.persistence.DaggerPersistenceComponent; import google.registry.tools.RegistryToolEnvironment; -import google.registry.util.Clock; import google.registry.util.NonFinalForTesting; -import google.registry.util.SystemClock; import java.util.Optional; import java.util.function.Supplier; @@ -44,9 +38,6 @@ public final class TransactionManagerFactory { /** Optional override to manually set the transaction manager per-test. */ private static Optional tmForTest = Optional.empty(); - /** The current clock (defined as a variable so we can override it in tests) */ - private static Clock clock = new SystemClock(); - /** Supplier for jpaTm so that it is initialized only once, upon first usage. */ @NonFinalForTesting private static Supplier jpaTm = @@ -56,8 +47,6 @@ public final class TransactionManagerFactory { private static Supplier replicaJpaTm = Suppliers.memoize(TransactionManagerFactory::createReplicaJpaTransactionManager); - private static boolean onBeam = false; - private TransactionManagerFactory() {} private static JpaTransactionManager createJpaTransactionManager() { @@ -102,17 +91,7 @@ public final class TransactionManagerFactory { * the migration schedule or the manually specified per-test transaction manager. */ public static TransactionManager tm() { - if (tmForTest.isPresent()) { - return tmForTest.get(); - } - if (onBeam) { - return jpaTm(); - } - return DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()) - .getPrimaryDatabase() - .equals(PrimaryDatabase.DATASTORE) - ? ofyTm() - : jpaTm(); + return tmForTest.orElseGet(TransactionManagerFactory::jpaTm); } /** @@ -174,7 +153,6 @@ public final class TransactionManagerFactory { public static void setJpaTmOnBeamWorker(Supplier jpaTmSupplier) { checkArgumentNotNull(jpaTmSupplier, "jpaTmSupplier"); jpaTm = Suppliers.memoize(jpaTmSupplier::get); - onBeam = true; } /** @@ -196,39 +174,4 @@ public final class TransactionManagerFactory { public static void removeTmOverrideForTest() { tmForTest = Optional.empty(); } - - public static void assertNotReadOnlyMode() { - if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()).isReadOnly()) { - throw new ReadOnlyModeException(); - } - } - - /** - * Asserts that async actions (contact/host deletes and host renames) are allowed. - * - *

These are allowed at all times except during the {@link - * DatabaseMigrationStateSchedule.MigrationState#DATASTORE_PRIMARY_NO_ASYNC} stage. Note that - * {@link ReadOnlyModeException} may well be thrown during other read-only stages inside the - * transaction manager; this method specifically checks only async actions. - */ - @DeleteAfterMigration - public static void assertAsyncActionsAreAllowed() { - if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()) - .equals(DATASTORE_PRIMARY_NO_ASYNC)) { - throw new ReadOnlyModeException(); - } - } - - /** Allows us to set the clock used by the factory in unit tests. */ - @VisibleForTesting - public static void setClockForTesting(Clock clock) { - TransactionManagerFactory.clock = clock; - } - - /** Registry is currently undergoing maintenance and is in read-only mode. */ - public static class ReadOnlyModeException extends IllegalStateException { - public ReadOnlyModeException() { - super("Registry is currently undergoing maintenance and is in read-only mode"); - } - } } diff --git a/core/src/main/java/google/registry/rde/RdeStagingMapper.java b/core/src/main/java/google/registry/rde/RdeStagingMapper.java index 9a153b398..d1885aa6d 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingMapper.java +++ b/core/src/main/java/google/registry/rde/RdeStagingMapper.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableSetMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import google.registry.model.EppResource; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DomainBase; import google.registry.model.host.HostResource; @@ -37,6 +38,7 @@ import java.util.function.Supplier; import org.joda.time.DateTime; /** Mapper for {@link RdeStagingAction}. */ +@DeleteAfterMigration public final class RdeStagingMapper extends Mapper { private static final long serialVersionUID = -1518185703789372524L; diff --git a/core/src/main/java/google/registry/rde/RdeStagingReducer.java b/core/src/main/java/google/registry/rde/RdeStagingReducer.java index b959a5677..96573c704 100644 --- a/core/src/main/java/google/registry/rde/RdeStagingReducer.java +++ b/core/src/main/java/google/registry/rde/RdeStagingReducer.java @@ -30,6 +30,7 @@ import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; import google.registry.keyring.api.KeyModule; import google.registry.keyring.api.PgpHelper; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.common.Cursor; import google.registry.model.rde.RdeMode; import google.registry.model.rde.RdeNamingUtils; @@ -59,6 +60,7 @@ import org.joda.time.DateTime; import org.joda.time.Duration; /** Reducer for {@link RdeStagingAction}. */ +@DeleteAfterMigration public final class RdeStagingReducer extends Reducer { private static final long serialVersionUID = 60326234579091203L; diff --git a/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java b/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java index f13d2b721..9d94ede32 100644 --- a/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java +++ b/core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java @@ -15,8 +15,6 @@ package google.registry.reporting.billing; import static google.registry.beam.BeamUtils.createJobName; -import static google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase.CLOUD_SQL; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; @@ -31,13 +29,11 @@ import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.persistence.PersistenceModule; import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Action.Service; import google.registry.request.Parameter; -import google.registry.request.RequestParameters; import google.registry.request.Response; import google.registry.request.auth.Auth; import google.registry.util.Clock; @@ -77,7 +73,6 @@ public class GenerateInvoicesAction implements Runnable { private final Clock clock; private final Response response; private final Dataflow dataflow; - private final PrimaryDatabase database; private final CloudTasksUtils cloudTasksUtils; @Inject @@ -88,7 +83,6 @@ public class GenerateInvoicesAction implements Runnable { @Config("billingBucketUrl") String billingBucketUrl, @Config("invoiceFilePrefix") String invoiceFilePrefix, @Parameter(BillingModule.PARAM_SHOULD_PUBLISH) boolean shouldPublish, - @Parameter(RequestParameters.PARAM_DATABASE) PrimaryDatabase database, YearMonth yearMonth, BillingEmailUtils emailUtils, CloudTasksUtils cloudTasksUtils, @@ -98,15 +92,9 @@ public class GenerateInvoicesAction implements Runnable { this.projectId = projectId; this.jobRegion = jobRegion; this.stagingBucketUrl = stagingBucketUrl; - // When generating the invoices using Cloud SQL before database cutover, save the reports in a - // separate bucket so that it does not overwrite the Datastore invoices. - if (tm().isOfy() && database.equals(CLOUD_SQL)) { - billingBucketUrl = billingBucketUrl.concat("-sql"); - } this.billingBucketUrl = billingBucketUrl; this.invoiceFilePrefix = invoiceFilePrefix; this.shouldPublish = shouldPublish; - this.database = database; this.yearMonth = yearMonth; this.emailUtils = emailUtils; this.cloudTasksUtils = cloudTasksUtils; @@ -129,7 +117,6 @@ public class GenerateInvoicesAction implements Runnable { new ImmutableMap.Builder() .put("yearMonth", yearMonth.toString("yyyy-MM")) .put("invoiceFilePrefix", invoiceFilePrefix) - .put("database", database.name()) .put("billingBucketUrl", billingBucketUrl) .put("registryEnvironment", RegistryEnvironment.get().name()) .put( diff --git a/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java b/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java index 8190ec75a..2ef242df5 100644 --- a/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java +++ b/core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java @@ -15,7 +15,6 @@ package google.registry.reporting.spec11; import static google.registry.beam.BeamUtils.createJobName; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.request.Action.Method.POST; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_OK; @@ -31,12 +30,10 @@ import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.config.RegistryEnvironment; import google.registry.keyring.api.KeyModule.Key; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.reporting.ReportingModule; import google.registry.request.Action; import google.registry.request.Action.Service; import google.registry.request.Parameter; -import google.registry.request.RequestParameters; import google.registry.request.Response; import google.registry.request.auth.Auth; import google.registry.util.Clock; @@ -73,7 +70,6 @@ public class GenerateSpec11ReportAction implements Runnable { private final Clock clock; private final Response response; private final Dataflow dataflow; - private final PrimaryDatabase database; private final boolean sendEmail; private final CloudTasksUtils cloudTasksUtils; @@ -85,7 +81,6 @@ public class GenerateSpec11ReportAction implements Runnable { @Config("reportingBucketUrl") String reportingBucketUrl, @Key("safeBrowsingAPIKey") String apiKey, @Parameter(ReportingModule.PARAM_DATE) LocalDate date, - @Parameter(RequestParameters.PARAM_DATABASE) PrimaryDatabase database, @Parameter(ReportingModule.SEND_EMAIL) boolean sendEmail, Clock clock, Response response, @@ -94,13 +89,9 @@ public class GenerateSpec11ReportAction implements Runnable { this.projectId = projectId; this.jobRegion = jobRegion; this.stagingBucketUrl = stagingBucketUrl; - if (tm().isOfy() && database.equals(PrimaryDatabase.CLOUD_SQL)) { - reportingBucketUrl = reportingBucketUrl.concat("-sql"); - } this.reportingBucketUrl = reportingBucketUrl; this.apiKey = apiKey; this.date = date; - this.database = database; this.clock = clock; this.response = response; this.dataflow = dataflow; @@ -121,8 +112,6 @@ public class GenerateSpec11ReportAction implements Runnable { ImmutableMap.of( "safeBrowsingApiKey", apiKey, - "database", - database.name(), ReportingModule.PARAM_DATE, date.toString(), "reportingBucketUrl", diff --git a/core/src/main/java/google/registry/request/RequestModule.java b/core/src/main/java/google/registry/request/RequestModule.java index 1209b6c1c..358c2de23 100644 --- a/core/src/main/java/google/registry/request/RequestModule.java +++ b/core/src/main/java/google/registry/request/RequestModule.java @@ -17,8 +17,6 @@ package google.registry.request; import static com.google.common.net.MediaType.JSON_UTF_8; import static google.registry.model.tld.Registries.assertTldExists; import static google.registry.model.tld.Registries.assertTldsExist; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.request.RequestParameters.extractOptionalParameter; import static google.registry.request.RequestParameters.extractRequiredParameter; import static google.registry.request.RequestParameters.extractSetOfParameters; import static java.nio.charset.StandardCharsets.UTF_8; @@ -33,7 +31,6 @@ import com.google.common.net.MediaType; import com.google.protobuf.ByteString; import dagger.Module; import dagger.Provides; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.request.HttpException.BadRequestException; import google.registry.request.HttpException.UnsupportedMediaTypeException; import google.registry.request.auth.AuthResult; @@ -83,15 +80,6 @@ public final class RequestModule { return tlds; } - @Provides - @Parameter(RequestParameters.PARAM_DATABASE) - static PrimaryDatabase provideDatabase(HttpServletRequest req) { - return extractOptionalParameter(req, RequestParameters.PARAM_DATABASE) - .map(String::toUpperCase) - .map(PrimaryDatabase::valueOf) - .orElse(tm().isOfy() ? PrimaryDatabase.DATASTORE : PrimaryDatabase.CLOUD_SQL); - } - @Provides static Response provideResponse(ResponseImpl response) { return response; diff --git a/core/src/main/java/google/registry/request/RequestParameters.java b/core/src/main/java/google/registry/request/RequestParameters.java index 4c2286f19..a5f2365e0 100644 --- a/core/src/main/java/google/registry/request/RequestParameters.java +++ b/core/src/main/java/google/registry/request/RequestParameters.java @@ -37,14 +37,6 @@ public final class RequestParameters { /** The standardized request parameter name used by any action taking multiple tld parameters. */ public static final String PARAM_TLDS = "tlds"; - /** - * The standardized optional request parameter name used by any action to specify which database - * to use, if the action supports such override. The supported values are (case-insensitive) - * "datastore" and "cloud_sql". - */ - // TODO (jianglai): delete this param after the database migration. - public static final String PARAM_DATABASE = "database"; - /** * Returns first GET or POST parameter associated with {@code name}. * diff --git a/core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java b/core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java deleted file mode 100644 index f9a5a4233..000000000 --- a/core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static com.google.common.base.Preconditions.checkState; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; - -import com.beust.jcommander.Parameters; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.OneTime; -import google.registry.model.domain.DomainBase; -import google.registry.persistence.VKey; - -/** - * Command to dedupe {@link BillingEvent.OneTime} entities having duplicate IDs. - * - *

This command is used to address the duplicate id issue we found for certain {@link - * BillingEvent.OneTime} entities. The command reassigns an application wide unique id to the - * problematic entity and resaves it, it also resaves the entity having reference to the problematic - * entity with the updated id. - * - *

To use this command, you will need to provide the path to a file containing a list of strings - * representing the literal of Objectify key for the problematic entities. An example key literal - * is: - * - * 

- * "DomainBase", "111111-TEST", "HistoryEntry", 2222222, "OneTime", 3333333
- *
- * - *

Note that the double quotes are part of the key literal. The key literal can be retrieved from - * the column __key__.path in BigQuery. - */ -@Parameters( - separators = " =", - commandDescription = "Dedupe BillingEvent.OneTime entities with duplicate IDs.") -public class DedupeOneTimeBillingEventIdsCommand extends ReadEntityFromKeyPathCommand { - - @Override - void process(OneTime entity) { - VKey key = entity.createVKey(); - VKey domainKey = getGrandParentAsDomain(key); - DomainBase domain = tm().transact(() -> tm().loadByKey(domainKey)); - - // The BillingEvent.OneTime entity to be resaved should be the billing event created a few - // years ago, so they should not be referenced from TransferData and GracePeriod in the domain. - assertNotInDomainTransferData(domain, key); - domain - .getGracePeriods() - .forEach( - gracePeriod -> - checkState( - !gracePeriod.getOneTimeBillingEvent().equals(key), - "Entity %s is referenced by a grace period in domain %s", - key, - domainKey)); - - // By setting id to 0L, Buildable.build() will assign an application wide unique id to it. - BillingEvent.OneTime uniqIdBillingEvent = entity.asBuilder().setId(0L).build(); - stageEntityKeyChange(entity, uniqIdBillingEvent); - } - - private static void assertNotInDomainTransferData(DomainBase domainBase, VKey key) { - if (!domainBase.getTransferData().isEmpty()) { - domainBase - .getTransferData() - .getServerApproveEntities() - .forEach( - entityKey -> - checkState( - !entityKey.equals(key), - "Entity %s is referenced by the transfer data in domain %s", - key, - domainBase.createVKey())); - } - } -} diff --git a/core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java b/core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java deleted file mode 100644 index 8a4c680f2..000000000 --- a/core/src/main/java/google/registry/tools/GetDatabaseMigrationStateCommand.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import com.beust.jcommander.Parameters; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationStateTransition; -import google.registry.model.common.TimedTransitionProperty; - -/** A command to check the current Registry 3.0 migration state of the database. */ -@DeleteAfterMigration -@Parameters(separators = " =", commandDescription = "Check current Registry 3.0 migration state") -public class GetDatabaseMigrationStateCommand implements CommandWithRemoteApi { - - @Override - public void run() throws Exception { - TimedTransitionProperty migrationSchedule = - DatabaseMigrationStateSchedule.get(); - System.out.println( - String.format("Current migration schedule: %s", migrationSchedule.toValueMap())); - } -} diff --git a/core/src/main/java/google/registry/tools/ListTxnsCommand.java b/core/src/main/java/google/registry/tools/ListTxnsCommand.java deleted file mode 100644 index b50f66a7f..000000000 --- a/core/src/main/java/google/registry/tools/ListTxnsCommand.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2022 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.base.Splitter; -import com.google.common.io.BaseEncoding; -import google.registry.model.common.Cursor; -import google.registry.persistence.transaction.Transaction; -import google.registry.persistence.transaction.TransactionEntity; -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; - -/** Lists {@link Cursor} timestamps used by locking rolling cursor tasks, like in RDE. */ -@Parameters(separators = " =", commandDescription = "Lists the contents of the Transaction table.") -final class ListTxnsCommand implements CommandWithRemoteApi { - - @Parameter(names = "--infile", description = "Parse an input file instead of reading from db.") - private String infile; - - @Parameter( - names = "--full_dump", - description = - "Do a full dump of the contents of the transaction. Without this, " - + "just write transactions as CSV lines suitable for ingestion via --infile.") - private boolean fullDump = false; - - @Override - public void run() { - if (infile == null) { - fetchFromDb(); - } else { - parseCsvFile(); - } - } - - private void parseCsvFile() { - try { - BufferedReader src = Files.newBufferedReader(Paths.get(infile), UTF_8); - String line; - while ((line = src.readLine()) != null) { - List cols = Splitter.on(",").splitToList(line); - writeRecord(Integer.parseInt(cols.get(0)), BaseEncoding.base64().decode(cols.get(1))); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void fetchFromDb() { - long lastTransactionId = 0; - List results; - - do { - final long txnId = lastTransactionId; // For use in the lambda. - results = - replicaJpaTm() - .transact( - () -> - replicaJpaTm() - .query( - "select t from TransactionEntity t where id > :lastTransactionId", - TransactionEntity.class) - .setParameter("lastTransactionId", txnId) - .setMaxResults(1000) - .getResultList()); - - for (TransactionEntity txn : results) { - writeRecord(txn.getId(), txn.getContents()); - lastTransactionId = txn.getId(); - } - } while (results.size() > 0); - } - - private void writeRecord(long id, byte[] contents) { - if (fullDump) { - Transaction txn; - try { - txn = Transaction.deserialize(contents); - } catch (IOException ex) { - System.err.printf("Error deserializing transaction %s\n", id); - return; - } - System.out.printf("transaction %s <<<\n", id); - for (Transaction.Mutation mut : txn.getMutations()) { - if (mut instanceof Transaction.Update) { - System.out.println("updating: " + ((Transaction.Update) mut).getEntity()); - } else { - System.out.println("deleting: " + ((Transaction.Delete) mut).getKey()); - } - } - System.out.println(">>>"); - } else { - System.out.printf("%s,%s\n", id, BaseEncoding.base64().encode(contents)); - } - } -} diff --git a/core/src/main/java/google/registry/tools/MutatingCommand.java b/core/src/main/java/google/registry/tools/MutatingCommand.java index d088c6f62..d34be38cd 100644 --- a/core/src/main/java/google/registry/tools/MutatingCommand.java +++ b/core/src/main/java/google/registry/tools/MutatingCommand.java @@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; import google.registry.model.ImmutableObject; -import google.registry.model.replay.SqlEntity; import google.registry.persistence.VKey; import java.util.ArrayList; import java.util.HashSet; @@ -90,12 +89,15 @@ public abstract class MutatingCommand extends ConfirmingCommand implements Comma ImmutableObject entity = MoreObjects.firstNonNull(oldEntity, newEntity); // This is one of the few cases where it is acceptable to create an asymmetric VKey (using - // createOfy()). We can use this code on DatastoreOnlyEntity's where we can't construct an + // createOfy()). We can use this code on datastore-only entities where we can't construct a // SQL key. - key = - entity instanceof SqlEntity - ? VKey.from(Key.create(entity)) - : VKey.createOfy(entity.getClass(), Key.create(entity)); + VKey createdKey; + try { + createdKey = VKey.from(Key.create(entity)); + } catch (RuntimeException e) { + createdKey = VKey.createOfy(entity.getClass(), Key.create(entity)); + } + key = createdKey; } /** diff --git a/core/src/main/java/google/registry/tools/RegistryTool.java b/core/src/main/java/google/registry/tools/RegistryTool.java index 81e1513a2..2b865f191 100644 --- a/core/src/main/java/google/registry/tools/RegistryTool.java +++ b/core/src/main/java/google/registry/tools/RegistryTool.java @@ -49,7 +49,6 @@ public final class RegistryTool { .put("create_reserved_list", CreateReservedListCommand.class) .put("create_tld", CreateTldCommand.class) .put("curl", CurlCommand.class) - .put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class) .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class) .put("delete_domain", DeleteDomainCommand.class) .put("delete_host", DeleteHostCommand.class) @@ -67,7 +66,6 @@ public final class RegistryTool { .put("get_allocation_token", GetAllocationTokenCommand.class) .put("get_claims_list", GetClaimsListCommand.class) .put("get_contact", GetContactCommand.class) - .put("get_database_migration_state", GetDatabaseMigrationStateCommand.class) .put("get_domain", GetDomainCommand.class) .put("get_history_entries", GetHistoryEntriesCommand.class) .put("get_host", GetHostCommand.class) @@ -94,7 +92,6 @@ public final class RegistryTool { .put("list_registrars", ListRegistrarsCommand.class) .put("list_reserved_lists", ListReservedListsCommand.class) .put("list_tlds", ListTldsCommand.class) - .put("list_txns", ListTxnsCommand.class) .put("load_snapshot", LoadSnapshotCommand.class) .put("load_test", LoadTestCommand.class) .put("lock_domain", LockDomainCommand.class) @@ -104,13 +101,11 @@ public final class RegistryTool { .put("registrar_contact", RegistrarContactCommand.class) .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class) .put("renew_domain", RenewDomainCommand.class) - .put("replay_txns", ReplayTxnsCommand.class) .put("resave_entities", ResaveEntitiesCommand.class) .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class) .put("resave_epp_resource", ResaveEppResourceCommand.class) .put("save_sql_credential", SaveSqlCredentialCommand.class) .put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class) - .put("set_database_migration_state", SetDatabaseMigrationStateCommand.class) .put("set_num_instances", SetNumInstancesCommand.class) .put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class) .put("setup_ote", SetupOteCommand.class) diff --git a/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java b/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java index c0fabc90f..cbdf04b53 100644 --- a/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java +++ b/core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java @@ -16,6 +16,7 @@ package google.registry.tools; import com.beust.jcommander.Parameters; import com.googlecode.objectify.Key; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.domain.DomainBase; import google.registry.persistence.VKey; import javax.annotation.Nullable; @@ -23,6 +24,7 @@ import org.joda.time.DateTime; /** Command to remove the Registry 1.0 key in {@link DomainBase} entity. */ @Parameters(separators = " =", commandDescription = "Remove .") +@DeleteAfterMigration public class RemoveRegistryOneKeyCommand extends ReadEntityFromKeyPathCommand { @Override diff --git a/core/src/main/java/google/registry/tools/ReplayTxnsCommand.java b/core/src/main/java/google/registry/tools/ReplayTxnsCommand.java deleted file mode 100644 index 25f683c0d..000000000 --- a/core/src/main/java/google/registry/tools/ReplayTxnsCommand.java +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2022 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import google.registry.persistence.transaction.Transaction; -import google.registry.persistence.transaction.TransactionEntity; -import java.util.List; - -@Parameters(separators = " =", commandDescription = "Replay a range of transactions.") -public class ReplayTxnsCommand implements CommandWithRemoteApi { - - private static final int BATCH_SIZE = 200; - - @Parameter( - names = {"-s", "--start-txn-id"}, - description = "Transaction id to start replaying at.", - required = true) - long startTxnId; - - @Override - public void run() throws Exception { - List txns; - do { - txns = - replicaJpaTm() - .transact( - () -> - replicaJpaTm() - .query( - "SELECT txn FROM TransactionEntity txn where id >= :startTxn ORDER" - + " BY id", - TransactionEntity.class) - .setParameter("startTxn", startTxnId) - .setMaxResults(BATCH_SIZE) - .getResultList()); - for (TransactionEntity txn : txns) { - System.out.println("Replaying transaction " + txn.getId()); - Transaction.deserialize(txn.getContents()); - startTxnId = txn.getId() + 1; - } - } while (txns.size() > 0); - } -} diff --git a/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java b/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java index 0052492c6..9363b9d41 100644 --- a/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java +++ b/core/src/main/java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java @@ -21,6 +21,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import com.beust.jcommander.Parameters; import com.googlecode.objectify.Key; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarContact; import google.registry.model.tld.Registry; @@ -32,6 +33,7 @@ import google.registry.model.tld.Registry; * {@link RegistrarContact}. */ @Parameters(commandDescription = "Re-save all environment entities.") +@DeleteAfterMigration final class ResaveEnvironmentEntitiesCommand implements CommandWithRemoteApi { private static final int BATCH_SIZE = 10; diff --git a/core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java b/core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java deleted file mode 100644 index f880eced6..000000000 --- a/core/src/main/java/google/registry/tools/SetDatabaseMigrationStateCommand.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.annotations.DeleteAfterMigration; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.tools.params.TransitionListParameter.MigrationStateTransitions; -import org.joda.time.DateTime; - -/** Command to set the Registry 3.0 database migration state schedule. */ -@DeleteAfterMigration -@Parameters( - separators = " =", - commandDescription = "Set the current database migration state schedule.") -public class SetDatabaseMigrationStateCommand extends ConfirmingCommand - implements CommandWithRemoteApi { - - private static final String WARNING_MESSAGE = - "Attempting to change the schedule with an effect that would take place within the next 10 " - + "minutes. The cache expiration duration is 5 minutes so this MAY BE DANGEROUS.\n"; - - @Parameter( - names = "--migration_schedule", - converter = MigrationStateTransitions.class, - validateWith = MigrationStateTransitions.class, - required = true, - description = - "Comma-delimited list of database transitions, of the form" - + "

A Datastore backup consists of an unsynchronized data export and a sequence of incremental - * Commit Logs that overlap with the export process. Together they can be used to recreate a - * consistent snapshot of the Datastore. - * - *

For convenience of test-writing, the {@link #fakeClock} is advanced by 1 millisecond after - * every transaction is invoked on this store, ensuring strictly increasing timestamps on causally - * dependent transactions. In production, the same ordering is ensured by sleep and retry. - */ -public final class BackupTestStore implements AutoCloseable { - - private static final DateTimeFormatter EXPORT_TIMESTAMP_FORMAT = - DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss_SSS"); - - private final FakeClock fakeClock; - private AppEngineExtension appEngine; - /** For fetching the persisted Datastore Entity directly. */ - private DatastoreService datastoreService; - - private CommitLogCheckpoint prevCommitLogCheckpoint; - - public BackupTestStore(FakeClock fakeClock) throws Exception { - this.fakeClock = fakeClock; - this.appEngine = - new AppEngineExtension.Builder() - .withDatastoreAndCloudSql() - .withoutCannedData() - .withClock(fakeClock) - .build(); - this.appEngine.setUp(); - datastoreService = DatastoreServiceFactory.getDatastoreService(); - } - - /** Returns the timestamp of the transaction. */ - long transact(Iterable deletes, Iterable newOrUpdated) { - long timestamp = fakeClock.nowUtc().getMillis(); - ofyTm() - .transact( - () -> { - auditedOfy().delete().entities(deletes); - auditedOfy().save().entities(newOrUpdated); - }); - fakeClock.advanceOneMilli(); - return timestamp; - } - - /** - * Inserts or updates {@code entities} in the Datastore and returns the timestamp of this - * transaction. - */ - @SafeVarargs - public final long insertOrUpdate(Object... entities) { - long timestamp = fakeClock.nowUtc().getMillis(); - ofyTm().transact(() -> auditedOfy().save().entities(entities).now()); - fakeClock.advanceOneMilli(); - return timestamp; - } - - /** Deletes {@code entities} from the Datastore and returns the timestamp of this transaction. */ - @SafeVarargs - public final long delete(Object... entities) { - long timestamp = fakeClock.nowUtc().getMillis(); - ofyTm().transact(() -> auditedOfy().delete().entities(entities).now()); - fakeClock.advanceOneMilli(); - return timestamp; - } - - /** - * Returns the persisted data that corresponds to {@code ofyEntity} as a Datastore {@link Entity}. - * - *

A typical use case for this method is in a test, when the caller has persisted newly created - * Objectify entity and want to find out the values of certain assign-on-persist properties. See - * {@link VersionedEntity} for more information. - */ - public Entity loadAsDatastoreEntity(Object ofyEntity) { - try { - return datastoreService.get(Key.create(ofyEntity).getRaw()); - } catch (EntityNotFoundException e) { - throw new NoSuchElementException(e.getMessage()); - } - } - - /** - * Returns the persisted data that corresponds to {@code ofyEntity} as an Objectify entity. - * - *

See {@link #loadAsDatastoreEntity} and {@link VersionedEntity} for more information. - */ - public ImmutableObject loadAsOfyEntity(ImmutableObject ofyEntity) { - try { - return auditedOfy().load().fromEntity(datastoreService.get(Key.create(ofyEntity).getRaw())); - } catch (EntityNotFoundException e) { - throw new NoSuchElementException(e.getMessage()); - } - } - /** - * Exports entities of the caller provided types and returns the directory where data is exported. - * - * @param exportRootPath path to the root directory of all exports. A subdirectory will be created - * for this export - * @param pojoTypes java class of all entities to be exported - * @param excludes {@link Set} of {@link Key keys} of the entities not to export.This can be used - * to simulate an inconsistent export - * @return directory where data is exported - */ - File export(String exportRootPath, Iterable> pojoTypes, Set> excludes) - throws IOException { - File exportDirectory = getExportDirectory(exportRootPath); - for (Class pojoType : pojoTypes) { - File perKindFile = - new File( - BackupPaths.getExportFileNameByShard( - exportDirectory.getAbsolutePath(), Key.getKind(pojoType), 0)); - checkState( - perKindFile.getParentFile().mkdirs(), - "Failed to create per-kind export directory for %s.", - perKindFile.getParentFile().getAbsolutePath()); - exportOneKind(perKindFile, pojoType, excludes); - } - return exportDirectory; - } - - private void exportOneKind(File perKindFile, Class pojoType, Set> excludes) - throws IOException { - LevelDbFileBuilder builder = new LevelDbFileBuilder(perKindFile); - for (Object pojo : auditedOfy().load().type(pojoType).iterable()) { - if (!excludes.contains(Key.create(pojo))) { - try { - // Must preserve UpdateTimestamp. Do not use auditedOfy().save().toEntity(pojo)! - builder.addEntity(datastoreService.get(Key.create(pojo).getRaw())); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - builder.build(); - } - - File saveCommitLogs(String commitLogDir) { - CommitLogCheckpoint checkpoint = CommitLogExports.computeCheckpoint(fakeClock); - File commitLogFile = - CommitLogExports.saveCommitLogs(commitLogDir, prevCommitLogCheckpoint, checkpoint); - prevCommitLogCheckpoint = checkpoint; - return commitLogFile; - } - - @Override - public void close() throws Exception { - if (appEngine != null) { - appEngine.tearDown(); - appEngine = null; - } - } - - private File getExportDirectory(String exportRootPath) { - File exportDirectory = - new File(exportRootPath, fakeClock.nowUtc().toString(EXPORT_TIMESTAMP_FORMAT)); - checkState( - exportDirectory.mkdirs(), - "Failed to create export directory %s.", - exportDirectory.getAbsolutePath()); - return exportDirectory; - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java b/core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java deleted file mode 100644 index fac1043ce..000000000 --- a/core/src/test/java/google/registry/beam/initsql/BackupTestStoreTest.java +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Streams; -import com.googlecode.objectify.Key; -import google.registry.backup.CommitLogImports; -import google.registry.backup.VersionedEntity; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DesignatedContact; -import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.VKey; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import google.registry.tools.LevelDbLogReader; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.stream.Stream; -import org.apache.beam.sdk.values.KV; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** Unit tests for {@link BackupTestStore}. */ -public class BackupTestStoreTest { - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - @TempDir File tempDir; - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension(); - - @RegisterExtension InjectExtension injectExtension = new InjectExtension(); - - private FakeClock fakeClock; - private BackupTestStore store; - - // Test data: - private Registry registry; - private ContactResource contact; - private DomainBase domain; - - @BeforeEach - void beforeEach() throws Exception { - fakeClock = new FakeClock(START_TIME); - store = new BackupTestStore(fakeClock); - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - registry = newRegistry("tld1", "TLD1"); - store.insertOrUpdate(registry); - - contact = newContactResource("contact_1"); - domain = newDomainBase("domain1.tld1", contact); - store.insertOrUpdate(contact, domain); - - // Save persisted data for assertions. - registry = (Registry) store.loadAsOfyEntity(registry); - contact = (ContactResource) store.loadAsOfyEntity(contact); - domain = (DomainBase) store.loadAsOfyEntity(domain); - } - - @AfterEach - void afterEach() throws Exception { - store.close(); - } - - @Test - void export_filesCreated() throws IOException { - String exportRootPath = tempDir.getAbsolutePath(); - assertThat(fakeClock.nowUtc().toString()).isEqualTo("2000-01-01T00:00:00.002Z"); - File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_002"); - assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile()) - .that(exportFolder.exists()) - .isFalse(); - File actualExportFolder = export(exportRootPath, ImmutableSet.of()); - assertThat(actualExportFolder).isEquivalentAccordingToCompareTo(exportFolder); - try (Stream files = - Files.walk(exportFolder.toPath()) - .filter(Files::isRegularFile) - .map(Path::toString) - .map(string -> string.substring(exportFolder.getAbsolutePath().length()))) { - assertThat(files) - .containsExactly( - "/all_namespaces/kind_Registry/output-0", - "/all_namespaces/kind_DomainBase/output-0", - "/all_namespaces/kind_ContactResource/output-0"); - } - } - - @Test - void export_folderNameChangesWithTime() throws IOException { - String exportRootPath = tempDir.getAbsolutePath(); - fakeClock.advanceOneMilli(); - File exportFolder = new File(exportRootPath, "2000-01-01T00:00:00_003"); - assertWithMessage("Directory %s should not exist.", exportFolder.getAbsoluteFile()) - .that(exportFolder.exists()) - .isFalse(); - assertThat(export(exportRootPath, ImmutableSet.of())) - .isEquivalentAccordingToCompareTo(exportFolder); - } - - @Test - void export_dataReadBack() throws IOException { - String exportRootPath = tempDir.getAbsolutePath(); - File exportFolder = export(exportRootPath, ImmutableSet.of()); - ImmutableList loadedRegistries = - loadExportedEntities(new File(exportFolder, "/all_namespaces/kind_Registry/output-0")); - assertThat(loadedRegistries).containsExactly(registry); - - ImmutableList loadedDomains = - loadExportedEntities(new File(exportFolder, "/all_namespaces/kind_DomainBase/output-0")); - assertThat(loadedDomains).containsExactly(domain); - - ImmutableList loadedContacts = - loadExportedEntities( - new File(exportFolder, "/all_namespaces/kind_ContactResource/output-0")); - assertThat(loadedContacts).containsExactly(contact); - } - - @Test - void export_excludeSomeEntity() throws IOException { - Registry newRegistry = newRegistry("tld2", "TLD2"); - store.insertOrUpdate(newRegistry); - newRegistry = (Registry) store.loadAsOfyEntity(newRegistry); - - String exportRootPath = tempDir.getAbsolutePath(); - File exportFolder = - export( - exportRootPath, ImmutableSet.of(Key.create(getCrossTldKey(), Registry.class, "tld1"))); - ImmutableList loadedRegistries = - loadExportedEntities(new File(exportFolder, "/all_namespaces/kind_Registry/output-0")); - assertThat(loadedRegistries).containsExactly(newRegistry); - } - - @Test - void saveCommitLogs_fileCreated() { - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - assertThat(commitLogFile.exists()).isTrue(); - assertThat(commitLogFile.getName()).isEqualTo("commit_diff_until_2000-01-01T00:00:00.002Z"); - } - - @Test - void saveCommitLogs_inserts() { - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - assertThat(commitLogFile.exists()).isTrue(); - ImmutableList mutations = CommitLogImports.loadEntities(commitLogFile); - InitSqlTestUtils.assertContainsExactlyElementsIn( - mutations, - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(registry)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(contact)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(domain))); - } - - @Test - void saveCommitLogs_deletes() { - fakeClock.advanceOneMilli(); - store.saveCommitLogs(tempDir.getAbsolutePath()); - ContactResource newContact = newContactResource("contact2"); - VKey vKey = newContact.createVKey(); - domain = - domain - .asBuilder() - .setRegistrant(vKey) - .setContacts( - ImmutableSet.of( - DesignatedContact.create(DesignatedContact.Type.ADMIN, vKey), - DesignatedContact.create(DesignatedContact.Type.TECH, vKey))) - .build(); - store.insertOrUpdate(domain, newContact); - store.delete(contact); - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - ImmutableList mutations = CommitLogImports.loadEntities(commitLogFile); - InitSqlTestUtils.assertContainsExactlyElementsIn( - mutations, - KV.of(fakeClock.nowUtc().getMillis() - 1, Key.create(contact).getRaw()), - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(domain)), - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(newContact))); - } - - @Test - void saveCommitLogs_empty() { - fakeClock.advanceOneMilli(); - store.saveCommitLogs(tempDir.getAbsolutePath()); - fakeClock.advanceOneMilli(); - File commitLogFile = store.saveCommitLogs(tempDir.getAbsolutePath()); - assertThat(commitLogFile.exists()).isTrue(); - assertThat(CommitLogImports.loadEntities(commitLogFile)).isEmpty(); - } - - private File export(String exportRootPath, ImmutableSet> excludes) throws IOException { - return store.export( - exportRootPath, - ImmutableList.of(ContactResource.class, DomainBase.class, Registry.class), - excludes); - } - - private static ImmutableList loadExportedEntities(File dataFile) throws IOException { - return Streams.stream(LevelDbLogReader.from(dataFile.toPath())) - .map(InitSqlTestUtils::bytesToEntity) - .map(InitSqlTestUtils::datastoreToOfyEntity) - .collect(ImmutableList.toImmutableList()); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java b/core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java deleted file mode 100644 index 49570ddd3..000000000 --- a/core/src/test/java/google/registry/beam/initsql/CommitLogTransformsTest.java +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import google.registry.backup.VersionedEntity; -import google.registry.beam.TestPipelineExtension; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.nio.file.Files; -import java.nio.file.Path; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** Unit tests for {@link Transforms} related to loading CommitLogs. */ -class CommitLogTransformsTest implements Serializable { - - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - private final FakeClock fakeClock = new FakeClock(START_TIME); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - private transient BackupTestStore store; - private File commitLogsDir; - private File firstCommitLogFile; - - // Canned data: - private transient Registry registry; - private transient ContactResource contact; - private transient DomainBase domain; - - @BeforeEach - void beforeEach() throws Exception { - store = new BackupTestStore(fakeClock); - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - registry = newRegistry("tld1", "TLD1"); - store.insertOrUpdate(registry); - contact = newContactResource("contact_1"); - domain = newDomainBase("domain1.tld1", contact); - store.insertOrUpdate(contact, domain); - - // Save persisted data for assertions. - registry = (Registry) store.loadAsOfyEntity(registry); - contact = (ContactResource) store.loadAsOfyEntity(contact); - domain = (DomainBase) store.loadAsOfyEntity(domain); - - commitLogsDir = Files.createDirectory(tmpDir.resolve("commit_logs")).toFile(); - firstCommitLogFile = store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - } - - @AfterEach - void afterEach() throws Exception { - if (store != null) { - store.close(); - store = null; - } - } - - @Test - void getCommitLogFilePatterns() { - PCollection patterns = - testPipeline.apply( - "Get CommitLog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())); - - ImmutableList expectedPatterns = - ImmutableList.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*"); - - PAssert.that(patterns).containsInAnyOrder(expectedPatterns); - - testPipeline.run(); - } - - @Test - void getFilesByPatterns() { - PCollection fileMetas = - testPipeline - .apply( - "File patterns to metadata", - Create.of(commitLogsDir.getAbsolutePath() + "/commit_diff_until_*") - .withCoder(StringUtf8Coder.of())) - .apply(Transforms.getFilesByPatterns()); - - // Transform fileMetas to file names for assertions. - PCollection fileNames = - fileMetas.apply( - "File metadata to path string", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement( - @Element Metadata metadata, OutputReceiver out) { - out.output(metadata.resourceId().toString()); - } - })); - - ImmutableList expectedFilenames = - ImmutableList.of(firstCommitLogFile.getAbsolutePath()); - - PAssert.that(fileNames).containsInAnyOrder(expectedFilenames); - - testPipeline.run(); - } - - @Test - void filterCommitLogsByTime() throws IOException { - ImmutableList commitLogFilenames = - ImmutableList.of( - "commit_diff_until_2000-01-01T00:00:00.000Z", - "commit_diff_until_2000-01-01T00:00:00.001Z", - "commit_diff_until_2000-01-01T00:00:00.002Z", - "commit_diff_until_2000-01-01T00:00:00.003Z", - "commit_diff_until_2000-01-01T00:00:00.004Z"); - - for (String name : commitLogFilenames) { - new File(commitLogsDir, name).createNewFile(); - } - - PCollection filteredFilenames = - testPipeline - .apply( - "Get commitlog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())) - .apply("Find commitlog files", Transforms.getFilesByPatterns()) - .apply( - "Filtered by Time", - Transforms.filterCommitLogsByTime( - DateTime.parse("2000-01-01T00:00:00.001Z"), - DateTime.parse("2000-01-01T00:00:00.003Z"))) - .apply( - "Extract path strings", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement( - @Element Metadata fileMeta, OutputReceiver out) { - out.output(fileMeta.resourceId().getFilename()); - } - })); - PAssert.that(filteredFilenames) - .containsInAnyOrder( - "commit_diff_until_2000-01-01T00:00:00.001Z", - "commit_diff_until_2000-01-01T00:00:00.002Z"); - - testPipeline.run(); - } - - @Test - void loadOneCommitLogFile() { - PCollection entities = - testPipeline - .apply( - "Get CommitLog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())) - .apply("Find CommitLogs", Transforms.getFilesByPatterns()) - .apply( - Transforms.loadCommitLogsFromFiles( - ImmutableSet.of("Registry", "ContactResource", "DomainBase"))); - - InitSqlTestUtils.assertContainsExactlyElementsIn( - entities, - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(registry)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(contact)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(domain))); - - testPipeline.run(); - } - - @Test - void loadOneCommitLogFile_filterByKind() { - PCollection entities = - testPipeline - .apply( - "Get CommitLog file patterns", - Transforms.getCommitLogFilePatterns(commitLogsDir.getAbsolutePath())) - .apply("Find CommitLogs", Transforms.getFilesByPatterns()) - .apply( - Transforms.loadCommitLogsFromFiles(ImmutableSet.of("Registry", "ContactResource"))); - - InitSqlTestUtils.assertContainsExactlyElementsIn( - entities, - KV.of(fakeClock.nowUtc().getMillis() - 2, store.loadAsDatastoreEntity(registry)), - KV.of(fakeClock.nowUtc().getMillis() - 1, store.loadAsDatastoreEntity(contact))); - - testPipeline.run(); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/DatastoreSetupHelper.java b/core/src/test/java/google/registry/beam/initsql/DatastoreSetupHelper.java deleted file mode 100644 index 750a49245..000000000 --- a/core/src/test/java/google/registry/beam/initsql/DatastoreSetupHelper.java +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.model.common.Cursor.CursorType.BRDA; -import static google.registry.model.common.Cursor.CursorType.RECURRING_BILLING; -import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE; -import static google.registry.testing.DatabaseHelper.newRegistry; -import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.testing.DatabaseHelper.persistSimpleResource; -import static google.registry.util.DateTimeUtils.END_OF_TIME; -import static google.registry.util.DateTimeUtils.START_OF_TIME; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.flows.domain.DomainFlowUtils; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.Flag; -import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.common.Cursor; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DesignatedContact; -import google.registry.model.domain.DomainAuthInfo; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.DomainHistory; -import google.registry.model.domain.GracePeriod; -import google.registry.model.domain.launch.LaunchNotice; -import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.domain.secdns.DelegationSignerData; -import google.registry.model.domain.token.AllocationToken; -import google.registry.model.eppcommon.AuthInfo.PasswordAuth; -import google.registry.model.eppcommon.StatusValue; -import google.registry.model.eppcommon.Trid; -import google.registry.model.host.HostResource; -import google.registry.model.poll.PollMessage; -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.tld.Registry; -import google.registry.model.transfer.DomainTransferData; -import google.registry.model.transfer.TransferStatus; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.FakeClock; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import org.joda.money.Money; - -/** - * Sets up a test scenario in Datastore. - * - *

The {@link #initializeData} populates Datastore with test data, including {@link DomainBase}, - * {@link DomainHistory}, and commit logs. The up-to-date version of the relevant entities are saved - * in public instance variables (e.g., {@link #domain} for easy access. - */ -public class DatastoreSetupHelper { - - /** - * All kinds of entities to be set up in the Datastore. Must contain all kinds known to {@link - * InitSqlPipeline}. - */ - public static final ImmutableList> ALL_KINDS = - ImmutableList.of( - Registry.class, - Cursor.class, - Registrar.class, - ContactResource.class, - RegistrarContact.class, - DomainBase.class, - HostResource.class, - HistoryEntry.class, - AllocationToken.class, - BillingEvent.Recurring.class, - BillingEvent.OneTime.class, - BillingEvent.Cancellation.class, - PollMessage.class); - - private final Path tmpDir; - private final FakeClock fakeClock; - - public File exportRootDir; - public File exportDir; - public File commitLogDir; - - public Registrar registrar1; - public Registrar registrar2; - public DomainBase domain; - public ContactResource contact1; - public ContactResource contact2; - public HostResource hostResource; - - public DomainHistory historyEntry; - - public Cursor globalCursor; - public Cursor tldCursor; - - public DatastoreSetupHelper(Path tempDir, FakeClock fakeClock) { - this.tmpDir = tempDir; - this.fakeClock = fakeClock; - } - - public DatastoreSetupHelper initializeData() throws Exception { - try (BackupTestStore store = new BackupTestStore(fakeClock)) { - exportRootDir = Files.createDirectory(tmpDir.resolve("exports")).toFile(); - - persistResource(newRegistry("com", "COM")); - registrar1 = persistResource(AppEngineExtension.makeRegistrar1()); - registrar2 = persistResource(AppEngineExtension.makeRegistrar2()); - Key domainKey = Key.create(null, DomainBase.class, "4-COM"); - hostResource = - persistResource( - new HostResource.Builder() - .setHostName("ns1.example.com") - .setSuperordinateDomain(VKey.from(domainKey)) - .setRepoId("1-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setPersistedCurrentSponsorRegistrarId(registrar2.getRegistrarId()) - .build()); - contact1 = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id1") - .setRepoId("2-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setPersistedCurrentSponsorRegistrarId(registrar2.getRegistrarId()) - .build()); - contact2 = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id2") - .setRepoId("3-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setPersistedCurrentSponsorRegistrarId(registrar1.getRegistrarId()) - .build()); - persistSimpleResource( - new RegistrarContact.Builder() - .setParent(registrar1) - .setName("John Abused") - .setEmailAddress("johnabuse@example.com") - .setVisibleInWhoisAsAdmin(true) - .setVisibleInWhoisAsTech(false) - .setPhoneNumber("+1.2125551213") - .setFaxNumber("+1.2125551213") - .setTypes(ImmutableSet.of(RegistrarContact.Type.ABUSE, RegistrarContact.Type.ADMIN)) - .build()); - historyEntry = - persistResource( - new DomainHistory.Builder() - .setDomainRepoId(domainKey.getName()) - .setModificationTime(fakeClock.nowUtc()) - .setRegistrarId(registrar1.getRegistrarId()) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .build()); - persistResource( - new AllocationToken.Builder().setToken("abc123").setTokenType(SINGLE_USE).build()); - Key historyEntryKey = Key.create(historyEntry); - BillingEvent.OneTime onetimeBillEvent = - new BillingEvent.OneTime.Builder() - .setId(1) - .setReason(Reason.RENEW) - .setTargetId("example.com") - .setRegistrarId("TheRegistrar") - .setCost(Money.parse("USD 44.00")) - .setPeriodYears(4) - .setEventTime(fakeClock.nowUtc()) - .setBillingTime(fakeClock.nowUtc()) - .setParent(historyEntryKey) - .build(); - persistResource(onetimeBillEvent); - Key oneTimeBillKey = Key.create(onetimeBillEvent); - BillingEvent.Recurring recurringBillEvent = - new BillingEvent.Recurring.Builder() - .setId(2) - .setReason(Reason.RENEW) - .setFlags(ImmutableSet.of(Flag.AUTO_RENEW)) - .setTargetId("example.com") - .setRegistrarId("TheRegistrar") - .setEventTime(fakeClock.nowUtc()) - .setRecurrenceEndTime(END_OF_TIME) - .setParent(historyEntryKey) - .build(); - persistResource(recurringBillEvent); - VKey recurringBillKey = recurringBillEvent.createVKey(); - PollMessage.Autorenew autorenewPollMessage = - new PollMessage.Autorenew.Builder() - .setId(3L) - .setTargetId("example.com") - .setRegistrarId("TheRegistrar") - .setEventTime(fakeClock.nowUtc()) - .setMsg("Domain was auto-renewed.") - .setParent(historyEntry) - .build(); - persistResource(autorenewPollMessage); - VKey autorenewPollKey = autorenewPollMessage.createVKey(); - PollMessage.OneTime oneTimePollMessage = - new PollMessage.OneTime.Builder() - .setId(1L) - .setParent(historyEntry) - .setEventTime(fakeClock.nowUtc()) - .setRegistrarId("TheRegistrar") - .setMsg(DomainFlowUtils.COLLISION_MESSAGE) - .build(); - persistResource(oneTimePollMessage); - VKey onetimePollKey = oneTimePollMessage.createVKey(); - domain = - persistResource( - new DomainBase.Builder() - .setDomainName("example.com") - .setRepoId("4-COM") - .setCreationRegistrarId(registrar1.getRegistrarId()) - .setLastEppUpdateTime(fakeClock.nowUtc()) - .setLastEppUpdateRegistrarId(registrar2.getRegistrarId()) - .setLastTransferTime(fakeClock.nowUtc()) - .setStatusValues( - ImmutableSet.of( - StatusValue.CLIENT_DELETE_PROHIBITED, - StatusValue.SERVER_DELETE_PROHIBITED, - StatusValue.SERVER_TRANSFER_PROHIBITED, - StatusValue.SERVER_UPDATE_PROHIBITED, - StatusValue.SERVER_RENEW_PROHIBITED, - StatusValue.SERVER_HOLD)) - .setRegistrant(contact1.createVKey()) - .setContacts( - ImmutableSet.of( - DesignatedContact.create( - DesignatedContact.Type.ADMIN, contact2.createVKey()))) - .setNameservers(ImmutableSet.of(hostResource.createVKey())) - .setSubordinateHosts(ImmutableSet.of("ns1.example.com")) - .setPersistedCurrentSponsorRegistrarId(registrar2.getRegistrarId()) - .setRegistrationExpirationTime(fakeClock.nowUtc().plusYears(1)) - .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("password"))) - .setDsData( - ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2}))) - .setLaunchNotice( - LaunchNotice.create("tcnid", "validatorId", START_OF_TIME, START_OF_TIME)) - .setTransferData( - new DomainTransferData.Builder() - .setGainingRegistrarId(registrar1.getRegistrarId()) - .setLosingRegistrarId(registrar2.getRegistrarId()) - .setPendingTransferExpirationTime(fakeClock.nowUtc()) - .setServerApproveEntities( - ImmutableSet.of( - VKey.from(oneTimeBillKey), recurringBillKey, autorenewPollKey)) - .setServerApproveBillingEvent(VKey.from(oneTimeBillKey)) - .setServerApproveAutorenewEvent(recurringBillKey) - .setServerApproveAutorenewPollMessage(autorenewPollKey) - .setTransferRequestTime(fakeClock.nowUtc().plusDays(1)) - .setTransferStatus(TransferStatus.SERVER_APPROVED) - .setTransferRequestTrid(Trid.create("client-trid", "server-trid")) - .build()) - .setDeletePollMessage(onetimePollKey) - .setAutorenewBillingEvent(recurringBillKey) - .setAutorenewPollMessage(autorenewPollKey) - .setSmdId("smdid") - .addGracePeriod( - GracePeriod.create( - GracePeriodStatus.ADD, - "4-COM", - fakeClock.nowUtc().plusDays(1), - "TheRegistrar", - null)) - .build()); - persistResource( - new BillingEvent.Cancellation.Builder() - .setReason(Reason.RENEW) - .setTargetId(domain.getDomainName()) - .setRegistrarId(domain.getCurrentSponsorRegistrarId()) - .setEventTime(fakeClock.nowUtc()) - .setBillingTime(fakeClock.nowUtc()) - .setRecurringEventKey(recurringBillEvent.createVKey()) - .setParent(historyEntryKey) - .build()); - globalCursor = persistResource(Cursor.createGlobal(RECURRING_BILLING, fakeClock.nowUtc())); - tldCursor = persistResource(Cursor.create(BRDA, fakeClock.nowUtc(), Registry.get("com"))); - exportDir = store.export(exportRootDir.getAbsolutePath(), ALL_KINDS, ImmutableSet.of()); - commitLogDir = Files.createDirectory(tmpDir.resolve("commits")).toFile(); - fakeClock.advanceOneMilli(); - } - return this; - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java b/core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java deleted file mode 100644 index 6e881729f..000000000 --- a/core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.testing.DatabaseHelper.cloneAndSetAutoTimestamps; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.google.appengine.api.datastore.Entity; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.OneTime; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DesignatedContact; -import google.registry.model.domain.DomainAuthInfo; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.DomainHistory; -import google.registry.model.domain.GracePeriod; -import google.registry.model.domain.launch.LaunchNotice; -import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.domain.secdns.DelegationSignerData; -import google.registry.model.eppcommon.AuthInfo.PasswordAuth; -import google.registry.model.eppcommon.StatusValue; -import google.registry.model.eppcommon.Trid; -import google.registry.model.host.HostResource; -import google.registry.model.ofy.Ofy; -import google.registry.model.poll.PollMessage; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.transfer.DomainTransferData; -import google.registry.model.transfer.TransferStatus; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import org.joda.time.Instant; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Unit tests for {@link DomainBaseUtil}. */ -public class DomainBaseUtilTest { - - private final FakeClock fakeClock = new FakeClock(Instant.now()); - - private DomainBase domain; - private Entity domainEntity; - private Key oneTimeBillKey; - private VKey recurringBillKey; - private Key domainKey; - - @RegisterExtension - AppEngineExtension appEngineExtension = - AppEngineExtension.builder().withDatastoreAndCloudSql().withClock(fakeClock).build(); - - @RegisterExtension InjectExtension injectExtension = new InjectExtension(); - - @BeforeEach - void beforeEach() { - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - createTld("com"); - domainKey = Key.create(null, DomainBase.class, "4-COM"); - VKey hostKey = - persistResource( - new HostResource.Builder() - .setHostName("ns1.example.com") - .setSuperordinateDomain(VKey.from(domainKey)) - .setRepoId("1-COM") - .build()) - .createVKey(); - VKey contact1Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id1") - .setRepoId("2-COM") - .build()) - .createVKey(); - VKey contact2Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id2") - .setRepoId("3-COM") - .build()) - .createVKey(); - Key historyEntryKey = - Key.create( - persistResource( - new DomainHistory.Builder() - .setDomainRepoId(domainKey.getName()) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .setRegistrarId("TheRegistrar") - .setModificationTime(fakeClock.nowUtc().minusYears(1)) - .build())); - oneTimeBillKey = Key.create(historyEntryKey, BillingEvent.OneTime.class, 1); - recurringBillKey = VKey.from(Key.create(historyEntryKey, BillingEvent.Recurring.class, 2)); - VKey autorenewPollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.Autorenew.class, 3)); - VKey onetimePollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.OneTime.class, 1)); - // Set up a new persisted domain entity. - domain = - persistResource( - cloneAndSetAutoTimestamps( - new DomainBase.Builder() - .setDomainName("example.com") - .setRepoId("4-COM") - .setCreationRegistrarId("a registrar") - .setLastEppUpdateTime(fakeClock.nowUtc()) - .setLastEppUpdateRegistrarId("AnotherRegistrar") - .setLastTransferTime(fakeClock.nowUtc()) - .setStatusValues( - ImmutableSet.of( - StatusValue.CLIENT_DELETE_PROHIBITED, - StatusValue.SERVER_DELETE_PROHIBITED, - StatusValue.SERVER_TRANSFER_PROHIBITED, - StatusValue.SERVER_UPDATE_PROHIBITED, - StatusValue.SERVER_RENEW_PROHIBITED, - StatusValue.SERVER_HOLD)) - .setRegistrant(contact1Key) - .setContacts( - ImmutableSet.of( - DesignatedContact.create(DesignatedContact.Type.ADMIN, contact2Key))) - .setNameservers(ImmutableSet.of(hostKey)) - .setSubordinateHosts(ImmutableSet.of("ns1.example.com")) - .setPersistedCurrentSponsorRegistrarId("losing") - .setRegistrationExpirationTime(fakeClock.nowUtc().plusYears(1)) - .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("password"))) - .setDsData( - ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2}))) - .setLaunchNotice( - LaunchNotice.create("tcnid", "validatorId", START_OF_TIME, START_OF_TIME)) - .setTransferData( - new DomainTransferData.Builder() - .setGainingRegistrarId("gaining") - .setLosingRegistrarId("losing") - .setPendingTransferExpirationTime(fakeClock.nowUtc()) - .setServerApproveEntities( - ImmutableSet.of( - VKey.from(oneTimeBillKey), recurringBillKey, autorenewPollKey)) - .setServerApproveBillingEvent(VKey.from(oneTimeBillKey)) - .setServerApproveAutorenewEvent(recurringBillKey) - .setServerApproveAutorenewPollMessage(autorenewPollKey) - .setTransferRequestTime(fakeClock.nowUtc().plusDays(1)) - .setTransferStatus(TransferStatus.SERVER_APPROVED) - .setTransferRequestTrid(Trid.create("client-trid", "server-trid")) - .build()) - .setDeletePollMessage(onetimePollKey) - .setAutorenewBillingEvent(recurringBillKey) - .setAutorenewPollMessage(autorenewPollKey) - .setSmdId("smdid") - .addGracePeriod( - GracePeriod.create( - GracePeriodStatus.ADD, - "4-COM", - fakeClock.nowUtc().plusDays(1), - "registrar", - null)) - .build())); - domainEntity = tm().transact(() -> auditedOfy().toEntity(domain)); - } - - @Test - void removeBillingAndPollAndHosts_allFkeysPresent() { - DomainBase domainTransformedByOfy = - domain - .asBuilder() - .setAutorenewBillingEvent(null) - .setAutorenewPollMessage(null) - .setNameservers(ImmutableSet.of()) - .setDeletePollMessage(null) - .setTransferData(null) - .setGracePeriods(ImmutableSet.of()) - .build(); - DomainBase domainTransformedByUtil = - (DomainBase) auditedOfy().toPojo(DomainBaseUtil.removeBillingAndPollAndHosts(domainEntity)); - // Compensates for the missing INACTIVE status. - domainTransformedByUtil = domainTransformedByUtil.asBuilder().build(); - assertAboutImmutableObjects() - .that(domainTransformedByUtil) - .isEqualExceptFields(domainTransformedByOfy, "revisions", "updateTimestamp"); - } - - @Test - void removeBillingAndPollAndHosts_noFkeysPresent() { - DomainBase domainWithoutFKeys = - domain - .asBuilder() - .setAutorenewBillingEvent(null) - .setAutorenewPollMessage(null) - .setNameservers(ImmutableSet.of()) - .setDeletePollMessage(null) - .setTransferData(null) - .setGracePeriods(ImmutableSet.of()) - .build(); - Entity entityWithoutFkeys = tm().transact(() -> auditedOfy().toEntity(domainWithoutFKeys)); - DomainBase domainTransformedByUtil = - (DomainBase) - auditedOfy().toPojo(DomainBaseUtil.removeBillingAndPollAndHosts(entityWithoutFkeys)); - // Compensates for the missing INACTIVE status. - domainTransformedByUtil = domainTransformedByUtil.asBuilder().build(); - assertAboutImmutableObjects() - .that(domainTransformedByUtil) - .isEqualExceptFields(domainWithoutFKeys, "revisions", "updateTimestamp"); - } - - @Test - void removeBillingAndPollAndHosts_notDomainBase() { - Entity contactEntity = - tm().transact(() -> auditedOfy().toEntity(DatabaseHelper.newContactResource("contact"))); - - assertThrows( - IllegalArgumentException.class, - () -> DomainBaseUtil.removeBillingAndPollAndHosts(contactEntity)); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/ExportLoadingTransformsTest.java b/core/src/test/java/google/registry/beam/initsql/ExportLoadingTransformsTest.java deleted file mode 100644 index ebcad614f..000000000 --- a/core/src/test/java/google/registry/beam/initsql/ExportLoadingTransformsTest.java +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.backup.VersionedEntity; -import google.registry.beam.TestPipelineExtension; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.File; -import java.io.Serializable; -import java.nio.file.Path; -import java.util.Collections; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.fs.MatchResult.Metadata; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** - * Unit tests for {@link Transforms} related to loading Datastore exports. - * - *

This class implements {@link Serializable} so that test {@link DoFn} classes may be inlined. - */ -class ExportLoadingTransformsTest implements Serializable { - - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - private static final ImmutableList> ALL_KINDS = - ImmutableList.of(Registry.class, ContactResource.class, DomainBase.class); - private static final ImmutableSet ALL_KIND_STRS = - ALL_KINDS.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet()); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - private FakeClock fakeClock; - private transient BackupTestStore store; - private File exportDir; - - // Canned data: - private transient Registry registry; - private transient ContactResource contact; - private transient DomainBase domain; - - @BeforeEach - void beforeEach() throws Exception { - fakeClock = new FakeClock(START_TIME); - store = new BackupTestStore(fakeClock); - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - registry = newRegistry("tld1", "TLD1"); - store.insertOrUpdate(registry); - - contact = newContactResource("contact_1"); - domain = newDomainBase("domain1.tld1", contact); - store.insertOrUpdate(contact, domain); - - // Save persisted data for assertions. - registry = (Registry) store.loadAsOfyEntity(registry); - contact = (ContactResource) store.loadAsOfyEntity(contact); - domain = (DomainBase) store.loadAsOfyEntity(domain); - - exportDir = store.export(tmpDir.toAbsolutePath().toString(), ALL_KINDS, Collections.EMPTY_SET); - } - - @AfterEach - void afterEach() throws Exception { - if (store != null) { - store.close(); - store = null; - } - } - - @Test - void getExportFilePatterns() { - PCollection patterns = - testPipeline.apply( - "Get Datastore file patterns", - Transforms.getDatastoreExportFilePatterns(exportDir.getAbsolutePath(), ALL_KIND_STRS)); - - ImmutableList expectedPatterns = - ImmutableList.of( - exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/output-*", - exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/output-*", - exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/output-*"); - - PAssert.that(patterns).containsInAnyOrder(expectedPatterns); - - testPipeline.run(); - } - - @Test - void getFilesByPatterns() { - PCollection fileMetas = - testPipeline - .apply( - "File patterns to metadata", - Create.of( - exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/output-*", - exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/output-*", - exportDir.getAbsolutePath() - + "/all_namespaces/kind_ContactResource/output-*") - .withCoder(StringUtf8Coder.of())) - .apply(Transforms.getFilesByPatterns()); - - // Transform fileMetas to file names for assertions. - PCollection fileNames = - fileMetas.apply( - "File metadata to path string", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement( - @Element Metadata metadata, OutputReceiver out) { - out.output(metadata.resourceId().toString()); - } - })); - - ImmutableList expectedFilenames = - ImmutableList.of( - exportDir.getAbsolutePath() + "/all_namespaces/kind_Registry/output-0", - exportDir.getAbsolutePath() + "/all_namespaces/kind_DomainBase/output-0", - exportDir.getAbsolutePath() + "/all_namespaces/kind_ContactResource/output-0"); - - PAssert.that(fileNames).containsInAnyOrder(expectedFilenames); - - testPipeline.run(); - } - - @Test - void loadDataFromFiles() { - PCollection entities = - testPipeline - .apply( - "Get Datastore file patterns", - Transforms.getDatastoreExportFilePatterns( - exportDir.getAbsolutePath(), ALL_KIND_STRS)) - .apply("Find Datastore files", Transforms.getFilesByPatterns()) - .apply("Load from Datastore files", Transforms.loadExportDataFromFiles()); - - InitSqlTestUtils.assertContainsExactlyElementsIn( - entities, - KV.of(Transforms.EXPORT_ENTITY_TIME_STAMP, store.loadAsDatastoreEntity(registry)), - KV.of(Transforms.EXPORT_ENTITY_TIME_STAMP, store.loadAsDatastoreEntity(contact)), - KV.of(Transforms.EXPORT_ENTITY_TIME_STAMP, store.loadAsDatastoreEntity(domain))); - - testPipeline.run(); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineGraphTest.java b/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineGraphTest.java deleted file mode 100644 index 7459c6f6a..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineGraphTest.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.truth.TextDiffSubject.assertWithMessageAboutUrlSource; - -import com.google.common.io.Resources; -import google.registry.beam.TestPipelineExtension; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.net.URL; -import org.apache.beam.runners.core.construction.renderer.PipelineDotRenderer; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Manages visualization of {@link InitSqlPipeline}. */ -class InitSqlPipelineGraphTest { - - private static final String GOLDEN_DOT_FILE = "pipeline_golden.dot"; - - private static final String[] OPTIONS_ARGS = - new String[] { - "--commitLogStartTimestamp=2000-01-01TZ", - "--commitLogEndTimestamp=2000-01-02TZ", - "--datastoreExportDir=/somedir", - "--commitLogDir=/someotherdir", - "--registryEnvironment=ALPHA" - }; - - private static final transient InitSqlPipelineOptions options = - PipelineOptionsFactory.fromArgs(OPTIONS_ARGS) - .withValidation() - .as(InitSqlPipelineOptions.class); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(false); - - @Test - void createPipeline_compareGraph() throws IOException { - new InitSqlPipeline(options).setupPipeline(testPipeline); - String dotString = PipelineDotRenderer.toDotString(testPipeline); - URL goldenDotUrl = Resources.getResource(InitSqlPipelineGraphTest.class, GOLDEN_DOT_FILE); - File outputFile = new File(new File(goldenDotUrl.getFile()).getParent(), "pipeline_curr.dot"); - try (PrintStream ps = new PrintStream(outputFile)) { - ps.print(dotString); - } - assertWithMessageAboutUrlSource( - "InitSqlPipeline graph changed. Run :core:updateInitSqlPipelineGraph to update.") - .that(outputFile.toURI().toURL()) - .hasSameContentAs(goldenDotUrl); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineOptionsTest.java b/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineOptionsTest.java deleted file mode 100644 index 299dabdc9..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineOptionsTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.jupiter.api.Test; - -/** Unit tests for {@link google.registry.beam.initsql.InitSqlPipelineOptions}. * */ -public class InitSqlPipelineOptionsTest { - - @Test - void registerToValidate() { - PipelineOptionsFactory.register(InitSqlPipelineOptions.class); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java b/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java deleted file mode 100644 index d08e69304..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlPipelineTest.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects; -import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; - -import google.registry.beam.TestPipelineExtension; -import google.registry.model.common.Cursor; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainBase; -import google.registry.model.host.HostResource; -import google.registry.model.ofy.Ofy; -import google.registry.model.registrar.Registrar; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.nio.file.Path; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.joda.time.DateTime; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** Unit tests for {@link InitSqlPipeline}. */ -class InitSqlPipelineTest { - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - private FakeClock fakeClock = new FakeClock(START_TIME); - - @RegisterExtension - @Order(Order.DEFAULT - 1) - final transient DatastoreEntityExtension datastore = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - @RegisterExtension - final transient JpaIntegrationTestExtension database = - new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationTestExtension(); - - DatastoreSetupHelper setupHelper; - - @BeforeEach - void beforeEach() throws Exception { - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - setupHelper = new DatastoreSetupHelper(tmpDir, fakeClock).initializeData(); - } - - @Test - void runPipeline() { - InitSqlPipelineOptions options = - PipelineOptionsFactory.fromArgs( - "--commitLogStartTimestamp=" + START_TIME, - "--commitLogEndTimestamp=" + fakeClock.nowUtc().plusMillis(1), - "--datastoreExportDir=" + setupHelper.exportDir.getAbsolutePath(), - "--commitLogDir=" + setupHelper.commitLogDir.getAbsolutePath()) - .withValidation() - .as(InitSqlPipelineOptions.class); - InitSqlPipeline initSqlPipeline = new InitSqlPipeline(options); - initSqlPipeline.run(testPipeline).waitUntilFinish(); - assertHostResourceEquals( - jpaTm().transact(() -> jpaTm().loadByKey(setupHelper.hostResource.createVKey())), - setupHelper.hostResource); - assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Registrar.class))) - .comparingElementsUsing(immutableObjectCorrespondence("lastUpdateTime")) - .containsExactly(setupHelper.registrar1, setupHelper.registrar2); - assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(ContactResource.class))) - .comparingElementsUsing(immutableObjectCorrespondence("revisions", "updateTimestamp")) - .containsExactly(setupHelper.contact1, setupHelper.contact2); - assertDomainEquals( - jpaTm().transact(() -> jpaTm().loadByKey(setupHelper.domain.createVKey())), - setupHelper.domain); - assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Cursor.class))) - .comparingElementsUsing(immutableObjectCorrespondence()) - .containsExactly(setupHelper.globalCursor, setupHelper.tldCursor); - } - - private static void assertHostResourceEquals(HostResource actual, HostResource expected) { - assertAboutImmutableObjects() - .that(actual) - .isEqualExceptFields(expected, "superordinateDomain", "revisions", "updateTimestamp"); - assertThat(actual.getSuperordinateDomain().getSqlKey()) - .isEqualTo(expected.getSuperordinateDomain().getSqlKey()); - } - - private static void assertDomainEquals(DomainBase actual, DomainBase expected) { - assertAboutImmutableObjects() - .that(actual) - .isEqualExceptFields( - expected, - "revisions", - "updateTimestamp", - "autorenewPollMessage", - "deletePollMessage", - "nsHosts", - "gracePeriods", - "transferData"); - assertThat(actual.getAdminContact().getSqlKey()) - .isEqualTo(expected.getAdminContact().getSqlKey()); - assertThat(actual.getRegistrant().getSqlKey()).isEqualTo(expected.getRegistrant().getSqlKey()); - assertThat(actual.getNsHosts()).isEqualTo(expected.getNsHosts()); - assertThat(actual.getAutorenewPollMessage().getOfyKey()) - .isEqualTo(expected.getAutorenewPollMessage().getOfyKey()); - assertThat(actual.getDeletePollMessage().getOfyKey()) - .isEqualTo(expected.getDeletePollMessage().getOfyKey()); - assertThat(actual.getUpdateTimestamp()).isEqualTo(expected.getUpdateTimestamp()); - // TODO(weiminyu): check gracePeriods and transferData when it is easier to do - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/InitSqlTestUtils.java b/core/src/test/java/google/registry/beam/initsql/InitSqlTestUtils.java deleted file mode 100644 index 1aed9d083..000000000 --- a/core/src/test/java/google/registry/beam/initsql/InitSqlTestUtils.java +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static org.apache.beam.sdk.values.TypeDescriptors.kvs; -import static org.apache.beam.sdk.values.TypeDescriptors.strings; - -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityTranslator; -import com.google.appengine.api.datastore.Key; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Streams; -import com.google.common.truth.Truth; -import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; -import google.registry.backup.VersionedEntity; -import java.io.Serializable; -import java.util.Collection; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.GroupByKey; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.TypeDescriptor; - -/** Test helpers for populating SQL with Datastore backups. */ -public final class InitSqlTestUtils { - - // Generates unique ids to distinguish reused transforms. - private static final AtomicInteger TRANSFORM_ID_GEN = new AtomicInteger(0); - - /** Converts a Datastore {@link Entity} to an Objectify entity. */ - public static Object datastoreToOfyEntity(Entity entity) { - return auditedOfy().load().fromEntity(entity); - } - - /** Serializes a Datastore {@link Entity} to byte array. */ - public static byte[] entityToBytes(Entity entity) { - return EntityTranslator.convertToPb(entity).toByteArray(); - } - - /** Deserializes raw bytes into {@link Entity}. */ - public static Entity bytesToEntity(byte[] bytes) { - EntityProto proto = new EntityProto(); - proto.parseFrom(bytes); - return EntityTranslator.createFromPb(proto); - } - - /** - * Asserts that the {@code actual} {@link Collection} of {@link VersionedEntity VersionedEntities} - * contains exactly the same elements in the {@code expected} array. - * - *

Each {@code expected} {@link KV key-value pair} refers to a versioned state of an Ofy - * entity. The {@link KV#getKey key} is the timestamp, while the {@link KV#getValue value} is - * either a Datastore {@link Entity} (for an existing entity) or a Datastore {@link Key} (for a - * deleted entity). - * - *

The {@Entity} instances in both actual and expected data are converted to Objectify entities - * so that value-equality checks can be performed. Datastore {@link Entity#equals Entity's equals - * method} only checks key-equality. - */ - @SafeVarargs - public static void assertContainsExactlyElementsIn( - Collection actual, KV... expected) { - assertThat(actual.stream().map(InitSqlTestUtils::rawEntityToOfyWithTimestamp)) - .containsExactlyElementsIn( - Stream.of(expected) - .map(InitSqlTestUtils::expectedToOfyWithTimestamp) - .collect(ImmutableList.toImmutableList())); - } - - /** - * Asserts that the {@code actual} {@link PCollection} of {@link VersionedEntity - * VersionedEntities} contains exactly the same elements in the {@code expected} array. - * - *

This method makes assertions in the pipeline and only use {@link PAssert} on the result. - * This way it supports assertions on Objectify entities, which {@code PAssert} cannot do ( since - * we have not implemented Coders for them). Compared with PAssert-compatible options like {@code - * google.registry.tools.EntityWrapper} or {@link EntityProto}, Objectify entities in Java give - * better-formatted error messages when assertions fail. - * - *

Each {@code expected} {@link KV key-value pair} refers to a versioned state of an Ofy - * entity. The {@link KV#getKey key} is the timestamp, while the {@link KV#getValue value} is - * either a Datastore {@link Entity} (for an existing entity) or a Datastore {@link Key} (for a - * deleted entity). - * - *

The {@Entity} instances in both actual and expected data are converted to Objectify entities - * so that value-equality checks can be performed. Datastore {@link Entity#equals Entity's equals - * method} only checks key-equality. - */ - @SafeVarargs - public static void assertContainsExactlyElementsIn( - PCollection actual, KV... expected) { - PCollection errMsgs = - actual - .apply( - "MapElements_" + TRANSFORM_ID_GEN.getAndIncrement(), - MapElements.into(kvs(strings(), TypeDescriptor.of(VersionedEntity.class))) - .via(rawEntity -> KV.of("The One Key", rawEntity))) - .apply("GroupByKey_" + TRANSFORM_ID_GEN.getAndIncrement(), GroupByKey.create()) - .apply( - "assertContainsExactlyElementsIn_" + TRANSFORM_ID_GEN.getAndIncrement(), - ParDo.of( - new DoFn>, String>() { - @ProcessElement - public void processElement( - @Element KV> input, - OutputReceiver out) { - ImmutableList> actual = - Streams.stream(input.getValue()) - .map(InitSqlTestUtils::rawEntityToOfyWithTimestamp) - .collect(ImmutableList.toImmutableList()); - try { - Truth.assertThat(actual) - .containsExactlyElementsIn( - Stream.of(expected) - .map(InitSqlTestUtils::expectedToOfyWithTimestamp) - .collect(ImmutableList.toImmutableList())); - } catch (AssertionError e) { - out.output(e.toString()); - } - } - })); - PAssert.that(errMsgs).empty(); - } - - private static KV rawEntityToOfyWithTimestamp(VersionedEntity rawEntity) { - return KV.of( - rawEntity.commitTimeMills(), - rawEntity.getEntity().map(InitSqlTestUtils::datastoreToOfyEntity).orElse(rawEntity.key())); - } - - private static KV expectedToOfyWithTimestamp(KV kv) { - return KV.of( - kv.getKey(), - kv.getValue() instanceof Key - ? kv.getValue() - : datastoreToOfyEntity((Entity) kv.getValue())); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/LoadDatastoreSnapshotTest.java b/core/src/test/java/google/registry/beam/initsql/LoadDatastoreSnapshotTest.java deleted file mode 100644 index 1d21b2867..000000000 --- a/core/src/test/java/google/registry/beam/initsql/LoadDatastoreSnapshotTest.java +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static google.registry.testing.DatabaseHelper.newContactResource; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newRegistry; - -import com.google.appengine.api.datastore.Entity; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.beam.TestPipelineExtension; -import google.registry.model.contact.ContactResource; -import google.registry.model.domain.DomainAuthInfo; -import google.registry.model.domain.DomainBase; -import google.registry.model.eppcommon.AuthInfo.PasswordAuth; -import google.registry.model.ofy.Ofy; -import google.registry.model.tld.Registry; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.joda.time.DateTime; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -/** - * Unit test for {@link Transforms#loadDatastoreSnapshot}. - * - *

The test setup involves three entities, one Registry, one Domain, and two Contacts. Events - * happen in the following order: - * - *

    - *
  1. Registry and a filler Contact are inserted to Datastore. - *
  2. A CommitLog is persisted. - *
  3. Registry is updated. - *
  4. Another Contact and Domain are inserted into Datastore. - *
  5. Datastore is exported, but misses the newly inserted Contact. - *
  6. Filler Contact is deleted. - *
  7. A second CommitLog is persisted. - *
  8. Domain is updated in the Datastore. - *
  9. The third and last CommitLog is persisted. - *
- * - * The final snapshot includes Registry, Domain, and Contact. This scenario verifies that: - * - *
    - *
  • Incremental changes committed before an export does not override the exported valie. - *
  • Entity missed by an export can be recovered from later CommitLogs. - *
  • Multiple changes to an entity is applied in order. - *
  • Deletes are properly handled. - *
- */ -class LoadDatastoreSnapshotTest { - - private static final DateTime START_TIME = DateTime.parse("2000-01-01T00:00:00.0Z"); - - private static final ImmutableList> ALL_KINDS = - ImmutableList.of(Registry.class, ContactResource.class, DomainBase.class); - private static final ImmutableSet ALL_KIND_STRS = - ALL_KINDS.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet()); - - @SuppressWarnings("WeakerAccess") - @TempDir - transient Path tmpDir; - - @RegisterExtension final transient InjectExtension injectExtension = new InjectExtension(); - - @RegisterExtension - final transient JpaIntegrationTestExtension jpaIntegrationTestExtension = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - - @RegisterExtension - @Order(value = 1) - final transient DatastoreEntityExtension datastoreEntityExtension = - new DatastoreEntityExtension().allThreads(true); - - @RegisterExtension - final transient TestPipelineExtension testPipeline = - TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); - - private FakeClock fakeClock; - private File exportRootDir; - private File exportDir; - private File commitLogsDir; - - // Canned data: - private transient Entity dsRegistry; - private transient Entity dsContact; - private transient Entity dsDomain; - - private transient DateTime registryLastUpdateTime; - private transient DateTime contactLastUpdateTime; - private transient DateTime domainLastUpdateTime; - - @BeforeEach - void beforeEach() throws Exception { - fakeClock = new FakeClock(START_TIME); - try (BackupTestStore store = new BackupTestStore(fakeClock)) { - injectExtension.setStaticField(Ofy.class, "clock", fakeClock); - - exportRootDir = Files.createDirectory(tmpDir.resolve("export_root")).toFile(); - commitLogsDir = Files.createDirectory(tmpDir.resolve("commit_logs")).toFile(); - - Registry registry = newRegistry("tld1", "TLD1"); - ContactResource fillerContact = newContactResource("contact_filler"); - store.insertOrUpdate(registry, fillerContact); - store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - - registry = - registry - .asBuilder() - .setCreateBillingCost(registry.getStandardCreateCost().plus(1.0d)) - .build(); - registryLastUpdateTime = fakeClock.nowUtc(); - store.insertOrUpdate(registry); - - ContactResource contact = newContactResource("contact"); - DomainBase domain = newDomainBase("domain1.tld1", contact); - contactLastUpdateTime = fakeClock.nowUtc(); - store.insertOrUpdate(contact, domain); - exportDir = - store.export( - exportRootDir.getAbsolutePath(), ALL_KINDS, ImmutableSet.of(Key.create(contact))); - - store.delete(fillerContact); - store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - - domain = - domain - .asBuilder() - .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("NewPass"))) - .build(); - domainLastUpdateTime = fakeClock.nowUtc(); - store.insertOrUpdate(domain); - store.saveCommitLogs(commitLogsDir.getAbsolutePath()); - - fakeClock.advanceOneMilli(); - - // Save persisted data for assertions. - dsRegistry = store.loadAsDatastoreEntity(registry); - dsContact = store.loadAsDatastoreEntity(contact); - dsDomain = store.loadAsDatastoreEntity(domain); - } - } - - @Test - void loadDatastoreSnapshot() { - PCollectionTuple snapshot = - testPipeline.apply( - Transforms.loadDatastoreSnapshot( - exportDir.getAbsolutePath(), - commitLogsDir.getAbsolutePath(), - START_TIME, - fakeClock.nowUtc(), - ALL_KIND_STRS)); - InitSqlTestUtils.assertContainsExactlyElementsIn( - snapshot.get(Transforms.createTagForKind("DomainBase")), - KV.of(domainLastUpdateTime.getMillis(), dsDomain)); - InitSqlTestUtils.assertContainsExactlyElementsIn( - snapshot.get(Transforms.createTagForKind("Registry")), - KV.of(registryLastUpdateTime.getMillis(), dsRegistry)); - InitSqlTestUtils.assertContainsExactlyElementsIn( - snapshot.get(Transforms.createTagForKind("ContactResource")), - KV.of(contactLastUpdateTime.getMillis(), dsContact)); - testPipeline.run(); - } -} diff --git a/core/src/test/java/google/registry/beam/initsql/TransformsTest.java b/core/src/test/java/google/registry/beam/initsql/TransformsTest.java deleted file mode 100644 index 3d80bdbda..000000000 --- a/core/src/test/java/google/registry/beam/initsql/TransformsTest.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.beam.initsql; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.beam.initsql.Transforms.repairBadData; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.newDomainBase; -import static google.registry.testing.DatabaseHelper.newHostResource; - -import com.google.appengine.api.datastore.Entity; -import google.registry.model.domain.DomainBase; -import google.registry.model.host.HostResource; -import google.registry.testing.AppEngineExtension; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Unit tests for {@link Transforms}. */ -public class TransformsTest { - - @RegisterExtension - public final AppEngineExtension appEngine = - AppEngineExtension.builder().withDatastoreAndCloudSql().build(); - - @BeforeEach - void beforeEach() { - createTld("tld"); - } - - @Test - void testRepairBadData_canonicalizesDomainName() { - DomainBase domain = newDomainBase("foobar.tld"); - Entity entity = ofyTm().transact(() -> auditedOfy().toEntity(domain)); - entity.setIndexedProperty("fullyQualifiedDomainName", "FOOBäR.TLD"); - assertThat(((DomainBase) auditedOfy().toPojo(repairBadData(entity))).getDomainName()) - .isEqualTo("xn--foobr-jra.tld"); - } - - @Test - void testRepairBadData_canonicalizesHostName() { - HostResource host = newHostResource("baz.foobar.tld"); - Entity entity = ofyTm().transact(() -> auditedOfy().toEntity(host)); - entity.setIndexedProperty( - "fullyQualifiedHostName", "b̴̹͔͓̣̭̫͇͕̻̬̱͇͗͌́̆̋͒a̶̬̖͚̋̈́̽̇͝͠z̵͠.FOOBäR.TLD"); - assertThat(((HostResource) auditedOfy().toPojo(repairBadData(entity))).getHostName()) - .isEqualTo( - "xn--baz-kdcb2ajgzb4jtg6doej4e6b9am7c7b6c5nd4k7gpa2a9a7dufyewec.xn--foobr-jra.tld"); - } -} diff --git a/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java b/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java index c5c24b9cd..df1905897 100644 --- a/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java +++ b/core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java @@ -51,7 +51,6 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; import google.registry.testing.TestDataHelper; -import google.registry.testing.TmOverrideExtension; import google.registry.util.ResourceUtils; import java.io.File; import java.nio.file.Files; @@ -95,10 +94,6 @@ class InvoicingPipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(new FakeClock()).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - @TempDir Path tmpDir; private static final String BILLING_BUCKET_URL = "billing_bucket"; @@ -282,7 +277,6 @@ class InvoicingPipelineTest { @Test void testSuccess_fullSqlPipeline() throws Exception { setupCloudSql(); - options.setDatabase("CLOUD_SQL"); InvoicingPipeline invoicingPipeline = new InvoicingPipeline(options); invoicingPipeline.setupPipeline(pipeline); pipeline.run(options).waitUntilFinish(); diff --git a/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java b/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java index 8371df291..29f7a32d3 100644 --- a/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java +++ b/core/src/test/java/google/registry/beam/rde/RdePipelineTest.java @@ -87,7 +87,6 @@ import google.registry.testing.CloudTasksHelper.TaskMatcher; import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; import google.registry.testing.FakeKeyringModule; -import google.registry.testing.TmOverrideExtension; import java.io.IOException; import java.util.function.Function; import java.util.regex.Matcher; @@ -166,10 +165,6 @@ public class RdePipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - @RegisterExtension final TestPipelineExtension pipeline = TestPipelineExtension.fromOptions(options).enableAbandonedNodeEnforcement(true); diff --git a/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java b/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java index f5f12978a..9714c1909 100644 --- a/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java +++ b/core/src/test/java/google/registry/beam/resave/ResaveAllEppResourcesPipelineTest.java @@ -42,7 +42,6 @@ import google.registry.persistence.transaction.JpaTransactionManager; import google.registry.persistence.transaction.TransactionManagerFactory; import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; -import google.registry.testing.TmOverrideExtension; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.joda.time.DateTime; import org.joda.time.Duration; @@ -70,10 +69,6 @@ public class ResaveAllEppResourcesPipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private final ResaveAllEppResourcesPipelineOptions options = PipelineOptionsFactory.create().as(ResaveAllEppResourcesPipelineOptions.class); diff --git a/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java b/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java index 60a8a2dd2..7b3c14d65 100644 --- a/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java +++ b/core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java @@ -50,7 +50,6 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; import google.registry.testing.FakeSleeper; -import google.registry.testing.TmOverrideExtension; import google.registry.util.ResourceUtils; import google.registry.util.Retrier; import java.io.File; @@ -128,10 +127,6 @@ class Spec11PipelineTest { final JpaIntegrationTestExtension database = new JpaTestExtensions.Builder().withClock(new FakeClock()).buildIntegrationTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private final Spec11PipelineOptions options = PipelineOptionsFactory.create().as(Spec11PipelineOptions.class); @@ -146,7 +141,6 @@ class Spec11PipelineTest { options.setDate(DATE); options.setSafeBrowsingApiKey(SAFE_BROWSING_API_KEY); options.setReportingBucketUrl(reportingBucketUrl.getAbsolutePath()); - options.setDatabase("DATASTORE"); threatMatches = pipeline.apply( Create.of( @@ -199,7 +193,6 @@ class Spec11PipelineTest { @Test void testSuccess_fullSqlPipeline() throws Exception { setupCloudSql(); - options.setDatabase("CLOUD_SQL"); EvaluateSafeBrowsingFn safeBrowsingFn = new EvaluateSafeBrowsingFn( SAFE_BROWSING_API_KEY, diff --git a/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java b/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java index 462c15f28..d90c119b4 100644 --- a/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java +++ b/core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java @@ -25,7 +25,6 @@ import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptio import static org.junit.jupiter.api.Assertions.assertThrows; import google.registry.flows.EppException; -import google.registry.flows.EppException.ReadOnlyModeEppException; import google.registry.flows.FlowUtils.NotLoggedInException; import google.registry.flows.ResourceFlowTestCase; import google.registry.flows.contact.ContactFlowUtils.BadInternationalizedPostalInfoException; @@ -33,10 +32,8 @@ import google.registry.flows.contact.ContactFlowUtils.DeclineContactDisclosureFi import google.registry.flows.exceptions.ResourceAlreadyExistsForThisClientException; import google.registry.flows.exceptions.ResourceCreateContentionException; import google.registry.model.contact.ContactResource; -import google.registry.testing.DatabaseHelper; import google.registry.testing.DualDatabaseTest; import google.registry.testing.TestOfyAndSql; -import google.registry.testing.TestOfyOnly; import org.joda.time.DateTime; /** Unit tests for {@link ContactCreateFlow}. */ @@ -137,12 +134,4 @@ class ContactCreateFlowTest extends ResourceFlowTestCase doFailingTest("domain_transfer_request.xml")); - assertAboutEppExceptions().that(thrown).marshalsToXml(); - DatabaseHelper.removeDatabaseMigrationSchedule(); - } } diff --git a/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java b/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java index 18904f7a6..e50f79e87 100644 --- a/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java +++ b/core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java @@ -59,7 +59,6 @@ import com.google.common.collect.ImmutableSortedMap; import com.googlecode.objectify.Key; import google.registry.config.RegistryConfig; import google.registry.flows.EppException; -import google.registry.flows.EppException.ReadOnlyModeEppException; import google.registry.flows.EppException.UnimplementedExtensionException; import google.registry.flows.EppRequestSource; import google.registry.flows.FlowUtils.NotLoggedInException; @@ -106,10 +105,8 @@ import google.registry.model.poll.PendingActionNotificationResponse.DomainPendin import google.registry.model.poll.PollMessage; import google.registry.model.tld.Registry; import google.registry.persistence.VKey; -import google.registry.testing.DatabaseHelper; import google.registry.testing.DualDatabaseTest; import google.registry.testing.TestOfyAndSql; -import google.registry.testing.TestOfyOnly; import java.util.Optional; import org.joda.money.Money; import org.joda.time.DateTime; @@ -1745,14 +1742,4 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase { @Test void testFailure_unknownRegistrar() { - deleteResource(getRegistrarBuilder().build()); + registrar.getContacts().forEach(DatabaseHelper::deleteResource); + deleteResource(registrar); doFailingTest("login_valid.xml", BadRegistrarIdException.class); } diff --git a/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java b/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java index c8719812b..958b85e1c 100644 --- a/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java +++ b/core/src/test/java/google/registry/mapreduce/inputs/ChildEntityInputTest.java @@ -42,6 +42,7 @@ import google.registry.model.domain.DomainHistory; import google.registry.model.index.EppResourceIndex; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -52,6 +53,7 @@ import java.util.Set; import org.joda.money.Money; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -60,6 +62,10 @@ class ChildEntityInputTest { private static final DateTime now = DateTime.now(DateTimeZone.UTC); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java b/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java index b28aa8083..a906934d8 100644 --- a/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java +++ b/core/src/test/java/google/registry/mapreduce/inputs/CommitLogManifestInputTest.java @@ -24,12 +24,14 @@ import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogManifest; import google.registry.testing.AppEngineExtension; import google.registry.testing.DatabaseHelper; +import google.registry.testing.TmOverrideExtension; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.NoSuchElementException; import java.util.Set; import org.joda.time.DateTime; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -44,6 +46,10 @@ final class CommitLogManifestInputTest { private static final DateTime DATE_TIME_NEW = DateTime.parse("2016-12-19T12:01Z"); private static final DateTime DATE_TIME_NEW2 = DateTime.parse("2017-12-19T12:00Z"); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java b/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java index ed2bf8e1d..d64ccc200 100644 --- a/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java +++ b/core/src/test/java/google/registry/mapreduce/inputs/EppResourceInputsTest.java @@ -37,6 +37,7 @@ import google.registry.model.domain.DomainBase; import google.registry.model.host.HostResource; import google.registry.model.index.EppResourceIndex; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -44,6 +45,7 @@ import java.io.ObjectOutputStream; import java.util.HashSet; import java.util.NoSuchElementException; import java.util.Set; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -52,6 +54,10 @@ class EppResourceInputsTest { private static final double EPSILON = 0.0001; + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java b/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java index bf87fd826..663ec6894 100644 --- a/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java +++ b/core/src/test/java/google/registry/model/CreateAutoTimestampTest.java @@ -22,7 +22,6 @@ import static org.joda.time.DateTimeZone.UTC; import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Ignore; import google.registry.model.common.CrossTldSingleton; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; import google.registry.testing.DualDatabaseTest; import google.registry.testing.TestOfyAndSql; @@ -43,7 +42,6 @@ public class CreateAutoTimestampTest { /** Timestamped class. */ @Entity(name = "CatTestEntity") - @EntityForTesting @javax.persistence.Entity public static class CreateAutoTimestampTestObject extends CrossTldSingleton { @Ignore @javax.persistence.Id long id = SINGLETON_ID; diff --git a/core/src/test/java/google/registry/model/ImmutableObjectTest.java b/core/src/test/java/google/registry/model/ImmutableObjectTest.java index a84905391..5c1c02ae3 100644 --- a/core/src/test/java/google/registry/model/ImmutableObjectTest.java +++ b/core/src/test/java/google/registry/model/ImmutableObjectTest.java @@ -19,7 +19,6 @@ import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Sets.newHashSet; import static com.google.common.truth.Truth.assertThat; import static google.registry.model.ImmutableObject.cloneEmptyToNull; -import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.util.DateTimeUtils.START_OF_TIME; import com.google.common.collect.ImmutableList; @@ -29,7 +28,6 @@ import com.google.common.collect.Iterables; import com.googlecode.objectify.Key; import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Id; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; import google.registry.util.CidrAddressBlock; import java.lang.reflect.Field; @@ -51,6 +49,7 @@ public class ImmutableObjectTest { public final AppEngineExtension appEngine = AppEngineExtension.builder() .withDatastoreAndCloudSql() + .withJpaUnitTestEntities(ValueObject.class) .withOfyTestEntities(ValueObject.class) .build(); @@ -279,10 +278,9 @@ public class ImmutableObjectTest { /** Simple subclass of ImmutableObject. */ @Entity - @EntityForTesting + @javax.persistence.Entity public static class ValueObject extends ImmutableObject { - @Id - long id; + @Id @javax.persistence.Id long id; String value; @@ -294,32 +292,6 @@ public class ImmutableObjectTest { } } - @Test - void testToHydratedString_skipsDoNotHydrate() { - RootObject root = new RootObject(); - root.hydrateMe = Key.create(persistResource(ValueObject.create(1, "foo"))); - root.skipMe = Key.create(persistResource(ValueObject.create(2, "bar"))); - String hydratedString = root.toHydratedString(); - assertThat(hydratedString).contains("foo"); - assertThat(hydratedString).doesNotContain("bar"); - } - - @Test - void testToHydratedString_expandsMaps() { - RootObject root = new RootObject(); - root.map = ImmutableMap.of("foo", Key.create(persistResource(ValueObject.create(1, "bar")))); - String hydratedString = root.toHydratedString(); - assertThat(hydratedString).contains("foo"); - assertThat(hydratedString).contains("bar"); - } - - @Test - void testToHydratedString_expandsCollections() { - RootObject root = new RootObject(); - root.set = ImmutableSet.of(Key.create(persistResource(ValueObject.create(1, "foo")))); - assertThat(root.toHydratedString()).contains("foo"); - } - @Test void testInsignificantFields() { HasInsignificantFields instance1 = diff --git a/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java b/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java index 9b5712305..69be69cc8 100644 --- a/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java +++ b/core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java @@ -23,7 +23,6 @@ import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Ignore; import google.registry.model.common.CrossTldSingleton; import google.registry.model.ofy.Ofy; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.VKey; import google.registry.testing.AppEngineExtension; import google.registry.testing.DualDatabaseTest; @@ -59,7 +58,6 @@ public class UpdateAutoTimestampTest { /** Timestamped class. */ @Entity(name = "UatTestEntity") @javax.persistence.Entity - @EntityForTesting public static class UpdateAutoTimestampTestObject extends CrossTldSingleton { @Ignore @javax.persistence.Id long id = SINGLETON_ID; UpdateAutoTimestamp updateTime = UpdateAutoTimestamp.create(null); diff --git a/core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java b/core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java deleted file mode 100644 index 8a0102325..000000000 --- a/core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.common; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_ONLY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_READ_ONLY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_ONLY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY; -import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY_READ_ONLY; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.EntityTestCase; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.model.domain.token.AllocationToken; -import google.registry.model.domain.token.AllocationToken.TokenType; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; -import google.registry.testing.DatabaseHelper; -import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** Tests for {@link DatabaseMigrationStateSchedule}. */ -public class DatabaseMigrationStateScheduleTest extends EntityTestCase { - - @BeforeEach - void beforeEach() { - fakeClock.setAutoIncrementByOneMilli(); - } - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @Test - void testEmpty_returnsDatastoreOnlyMap() { - assertThat(DatabaseMigrationStateSchedule.getUncached()) - .isEqualTo(DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP); - } - - @Test - void testValidTransitions() { - // First, verify that no-ops are safe - for (MigrationState migrationState : MigrationState.values()) { - runValidTransition(migrationState, migrationState); - } - - // Next, the transitions that will actually cause a change - runValidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY); - - runValidTransition(DATASTORE_PRIMARY, DATASTORE_ONLY); - runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC); - runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY); - - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_ONLY); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_NO_ASYNC); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY); - - runValidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_READ_ONLY); - runValidTransition(SQL_PRIMARY_READ_ONLY, SQL_PRIMARY); - - runValidTransition(SQL_PRIMARY, SQL_PRIMARY_READ_ONLY); - runValidTransition(SQL_PRIMARY, SQL_ONLY); - - runValidTransition(SQL_ONLY, SQL_PRIMARY); - } - - @Test - void testInvalidTransitions() { - runInvalidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY); - runInvalidTransition(DATASTORE_ONLY, SQL_ONLY); - - runInvalidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY_READ_ONLY); - runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY); - runInvalidTransition(DATASTORE_PRIMARY, SQL_ONLY); - - runInvalidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_ONLY); - - runInvalidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_ONLY); - runInvalidTransition(SQL_PRIMARY_READ_ONLY, DATASTORE_PRIMARY); - runInvalidTransition(SQL_PRIMARY_READ_ONLY, SQL_ONLY); - - runInvalidTransition(SQL_PRIMARY, DATASTORE_ONLY); - runInvalidTransition(SQL_PRIMARY, DATASTORE_PRIMARY); - runInvalidTransition(SQL_PRIMARY, DATASTORE_PRIMARY_READ_ONLY); - - runInvalidTransition(SQL_ONLY, DATASTORE_ONLY); - runInvalidTransition(SQL_ONLY, DATASTORE_PRIMARY); - runInvalidTransition(SQL_ONLY, DATASTORE_PRIMARY_READ_ONLY); - } - - @Test - void testFailure_newMapImpliesInvalidChangeNow() { - DateTime startTime = fakeClock.nowUtc(); - fakeClock.advanceBy(Duration.standardHours(6)); - - // The new map is valid by itself, but not with the current state of DATASTORE_ONLY because the - // new map implies that the current state is DATASTORE_PRIMARY_READ_ONLY - ImmutableSortedMap nowInvalidMap = - ImmutableSortedMap.naturalOrder() - .put(START_OF_TIME, DATASTORE_ONLY) - .put(startTime.plusHours(1), DATASTORE_PRIMARY) - .put(startTime.plusHours(2), DATASTORE_PRIMARY_NO_ASYNC) - .put(startTime.plusHours(3), DATASTORE_PRIMARY_READ_ONLY) - .build(); - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(nowInvalidMap))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "Cannot transition from current state-as-of-now DATASTORE_ONLY " - + "to new state-as-of-now DATASTORE_PRIMARY_READ_ONLY"); - } - - @Test - void testFailure_notInTransaction() { - IllegalStateException thrown = - assertThrows( - IllegalStateException.class, - () -> - DatabaseMigrationStateSchedule.set( - DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP.toValueMap())); - assertThat(thrown).hasMessageThat().isEqualTo("Not in a transaction"); - } - - @Test - void testSuccess_factoryUsesSchedule() { - assertThat(tm().isOfy()).isTrue(); - // set the schedule to have converted to SQL_PRIMARY in the past - fakeClock.setTo(START_OF_TIME.plusDays(1)); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY); - assertThat(tm().isOfy()).isFalse(); - } - - @Test - void testSuccess_factoryUsesReadOnly() { - createTld("tld"); - fakeClock.setTo(START_OF_TIME.plusDays(1)); - AllocationToken token = - new AllocationToken.Builder().setToken("token").setTokenType(TokenType.SINGLE_USE).build(); - runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC); - runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY); - assertThrows(ReadOnlyModeException.class, () -> persistResource(token)); - runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY); - assertThrows(ReadOnlyModeException.class, () -> persistResource(token)); - runValidTransition(SQL_PRIMARY_READ_ONLY, SQL_PRIMARY); - persistResource(token); - } - - private void runValidTransition(MigrationState from, MigrationState to) { - ImmutableSortedMap transitions = - createMapEndingWithTransition(from, to); - jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions)); - assertThat(DatabaseMigrationStateSchedule.getUncached().toValueMap()) - .containsExactlyEntriesIn(transitions); - } - - private void runInvalidTransition(MigrationState from, MigrationState to) { - ImmutableSortedMap transitions = - createMapEndingWithTransition(from, to); - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - String.format("validStateTransitions map cannot transition from %s to %s.", from, to)); - } - - // Create a transition map that is valid up to the "from" transition, then add the "to" transition - private ImmutableSortedMap createMapEndingWithTransition( - MigrationState from, MigrationState to) { - ImmutableSortedMap.Builder builder = - ImmutableSortedMap.naturalOrder(); - builder.put(START_OF_TIME, DATASTORE_ONLY); - MigrationState[] allMigrationStates = MigrationState.values(); - for (int i = 0; i < allMigrationStates.length; i++) { - builder.put(fakeClock.nowUtc().plusMinutes(i), allMigrationStates[i]); - if (allMigrationStates[i].equals(from)) { - break; - } - } - builder.put(fakeClock.nowUtc().plusDays(1), to); - return builder.build(); - } -} diff --git a/core/src/test/java/google/registry/model/domain/DomainBaseTest.java b/core/src/test/java/google/registry/model/domain/DomainBaseTest.java index 936bf8dc6..a92d69a13 100644 --- a/core/src/test/java/google/registry/model/domain/DomainBaseTest.java +++ b/core/src/test/java/google/registry/model/domain/DomainBaseTest.java @@ -23,8 +23,12 @@ import static google.registry.testing.DatabaseHelper.cloneAndSetAutoTimestamps; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.newDomainBase; import static google.registry.testing.DatabaseHelper.newHostResource; +import static google.registry.testing.DatabaseHelper.persistActiveContact; +import static google.registry.testing.DatabaseHelper.persistActiveDomain; +import static google.registry.testing.DatabaseHelper.persistActiveHost; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DomainBaseSubject.assertAboutDomains; +import static google.registry.util.DateTimeUtils.END_OF_TIME; import static google.registry.util.DateTimeUtils.START_OF_TIME; import static org.joda.money.CurrencyUnit.USD; import static org.joda.time.DateTimeZone.UTC; @@ -41,6 +45,7 @@ import google.registry.model.EntityTestCase; import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObjectSubject; import google.registry.model.billing.BillingEvent; +import google.registry.model.billing.BillingEvent.Flag; import google.registry.model.billing.BillingEvent.Reason; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DesignatedContact.Type; @@ -70,50 +75,55 @@ public class DomainBaseTest extends EntityTestCase { private DomainBase domain; private VKey oneTimeBillKey; private VKey recurringBillKey; - private Key historyEntryKey; + private DomainHistory domainHistory; private VKey contact1Key, contact2Key; @BeforeEach void setUp() { createTld("com"); - VKey domainKey = VKey.from(Key.create(null, DomainBase.class, "4-COM")); - VKey hostKey = + domain = persistActiveDomain("example.com"); + VKey hostKey = persistActiveHost("ns1.example.com").createVKey(); + contact1Key = persistActiveContact("contact_id1").createVKey(); + contact2Key = persistActiveContact("contact_id1").createVKey(); + domainHistory = persistResource( - new HostResource.Builder() - .setHostName("ns1.example.com") - .setSuperordinateDomain(domainKey) - .setRepoId("1-COM") + new DomainHistory.Builder() + .setDomainRepoId(domain.createVKey().getOfyKey().getName()) + .setModificationTime(fakeClock.nowUtc()) + .setType(HistoryEntry.Type.DOMAIN_CREATE) + .setRegistrarId("TheRegistrar") + .build()); + oneTimeBillKey = + persistResource( + new BillingEvent.OneTime.Builder() + // Use SERVER_STATUS so we don't have to add a period. + .setReason(Reason.SERVER_STATUS) + .setTargetId(domain.getDomainName()) + .setRegistrarId(domain.getCurrentSponsorRegistrarId()) + .setDomainRepoId(domain.getRepoId()) + .setBillingTime(DateTime.now(UTC)) + .setCost(Money.of(USD, 100)) + .setEventTime(DateTime.now(UTC).plusYears(1)) + .setParent(domainHistory) .build()) .createVKey(); - contact1Key = + recurringBillKey = persistResource( - new ContactResource.Builder() - .setContactId("contact_id1") - .setRepoId("2-COM") + new BillingEvent.Recurring.Builder() + .setReason(Reason.RENEW) + .setFlags(ImmutableSet.of(Flag.AUTO_RENEW)) + .setTargetId(domain.getDomainName()) + .setRegistrarId(domain.getCurrentSponsorRegistrarId()) + .setDomainRepoId(domain.getRepoId()) + .setEventTime(DateTime.now(UTC).plusYears(1)) + .setRecurrenceEndTime(END_OF_TIME) + .setParent(domainHistory) .build()) .createVKey(); - contact2Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id2") - .setRepoId("3-COM") - .build()) - .createVKey(); - historyEntryKey = - Key.create( - persistResource( - new DomainHistory.Builder() - .setDomainRepoId(domainKey.getOfyKey().getName()) - .setModificationTime(fakeClock.nowUtc()) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .setRegistrarId("aregistrar") - .build())); - oneTimeBillKey = VKey.from(Key.create(historyEntryKey, BillingEvent.OneTime.class, 1)); - recurringBillKey = VKey.from(Key.create(historyEntryKey, BillingEvent.Recurring.class, 2)); VKey autorenewPollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.Autorenew.class, 3)); + VKey.from(Key.create(Key.create(domainHistory), PollMessage.Autorenew.class, 3)); VKey onetimePollKey = - VKey.from(Key.create(historyEntryKey, PollMessage.OneTime.class, 1)); + VKey.from(Key.create(Key.create(domainHistory), PollMessage.OneTime.class, 1)); // Set up a new persisted domain entity. domain = persistResource( @@ -121,9 +131,9 @@ public class DomainBaseTest extends EntityTestCase { new DomainBase.Builder() .setDomainName("example.com") .setRepoId("4-COM") - .setCreationRegistrarId("aregistrar") + .setCreationRegistrarId("TheRegistrar") .setLastEppUpdateTime(fakeClock.nowUtc()) - .setLastEppUpdateRegistrarId("AnotherRegistrar") + .setLastEppUpdateRegistrarId("NewRegistrar") .setLastTransferTime(fakeClock.nowUtc()) .setStatusValues( ImmutableSet.of( @@ -137,7 +147,7 @@ public class DomainBaseTest extends EntityTestCase { .setContacts(ImmutableSet.of(DesignatedContact.create(Type.ADMIN, contact2Key))) .setNameservers(ImmutableSet.of(hostKey)) .setSubordinateHosts(ImmutableSet.of("ns1.example.com")) - .setPersistedCurrentSponsorRegistrarId("losing") + .setPersistedCurrentSponsorRegistrarId("NewRegistrar") .setRegistrationExpirationTime(fakeClock.nowUtc().plusYears(1)) .setAuthInfo(DomainAuthInfo.create(PasswordAuth.create("password"))) .setDsData( @@ -146,8 +156,8 @@ public class DomainBaseTest extends EntityTestCase { LaunchNotice.create("tcnid", "validatorId", START_OF_TIME, START_OF_TIME)) .setTransferData( new DomainTransferData.Builder() - .setGainingRegistrarId("gaining") - .setLosingRegistrarId("losing") + .setGainingRegistrarId("TheRegistrar") + .setLosingRegistrarId("NewRegistrar") .setPendingTransferExpirationTime(fakeClock.nowUtc()) .setServerApproveEntities( ImmutableSet.of(oneTimeBillKey, recurringBillKey, autorenewPollKey)) @@ -165,10 +175,10 @@ public class DomainBaseTest extends EntityTestCase { .addGracePeriod( GracePeriod.create( GracePeriodStatus.ADD, - "4-COM", + domain.getRepoId(), fakeClock.nowUtc().plusDays(1), - "registrar", - null)) + "TheRegistrar", + oneTimeBillKey)) .setAutorenewEndTime(Optional.of(fakeClock.nowUtc().plusYears(2))) .setDnsRefreshRequestTime(Optional.of(fakeClock.nowUtc())) .build())); @@ -192,28 +202,15 @@ public class DomainBaseTest extends EntityTestCase { @Test void testVKeyRestoration() { - assertThat(domain.deletePollMessageHistoryId).isEqualTo(historyEntryKey.getId()); - assertThat(domain.autorenewBillingEventHistoryId).isEqualTo(historyEntryKey.getId()); - assertThat(domain.autorenewPollMessageHistoryId).isEqualTo(historyEntryKey.getId()); + assertThat(domain.deletePollMessageHistoryId).isEqualTo(domainHistory.getId()); + assertThat(domain.autorenewBillingEventHistoryId).isEqualTo(domainHistory.getId()); + assertThat(domain.autorenewPollMessageHistoryId).isEqualTo(domainHistory.getId()); assertThat(domain.getTransferData().getServerApproveBillingEventHistoryId()) - .isEqualTo(historyEntryKey.getId()); + .isEqualTo(domainHistory.getId()); assertThat(domain.getTransferData().getServerApproveAutorenewEventHistoryId()) - .isEqualTo(historyEntryKey.getId()); + .isEqualTo(domainHistory.getId()); assertThat(domain.getTransferData().getServerApproveAutorenewPollMessageHistoryId()) - .isEqualTo(historyEntryKey.getId()); - } - - @Test - void testIndexing() throws Exception { - verifyDatastoreIndexing( - domain, - "allContacts.contact", - "fullyQualifiedDomainName", - "nsHosts", - "currentSponsorClientId", - "deletionTime", - "tld", - "autorenewEndTime"); + .isEqualTo(domainHistory.getId()); } @Test @@ -366,7 +363,7 @@ public class DomainBaseTest extends EntityTestCase { VKey newAutorenewEvent) { assertThat(domain.getTransferData().getTransferStatus()) .isEqualTo(TransferStatus.SERVER_APPROVED); - assertThat(domain.getCurrentSponsorRegistrarId()).isEqualTo("winner"); + assertThat(domain.getCurrentSponsorRegistrarId()).isEqualTo("TheRegistrar"); assertThat(domain.getLastTransferTime()).isEqualTo(fakeClock.nowUtc().plusDays(1)); assertThat(domain.getRegistrationExpirationTime()).isEqualTo(newExpirationTime); assertThat(domain.getAutorenewBillingEvent()).isEqualTo(newAutorenewEvent); @@ -374,18 +371,19 @@ public class DomainBaseTest extends EntityTestCase { private void doExpiredTransferTest(DateTime oldExpirationTime) { DomainHistory historyEntry = - new DomainHistory.Builder() - .setDomain(domain) - .setModificationTime(fakeClock.nowUtc()) - .setRegistrarId(domain.getCurrentSponsorRegistrarId()) - .setType(HistoryEntry.Type.DOMAIN_TRANSFER_REQUEST) - .build(); + persistResource( + new DomainHistory.Builder() + .setDomain(domain) + .setModificationTime(fakeClock.nowUtc()) + .setRegistrarId(domain.getCurrentSponsorRegistrarId()) + .setType(HistoryEntry.Type.DOMAIN_TRANSFER_REQUEST) + .build()); BillingEvent.OneTime transferBillingEvent = persistResource( new BillingEvent.OneTime.Builder() .setReason(Reason.TRANSFER) - .setRegistrarId("winner") - .setTargetId("example.com") + .setRegistrarId("TheRegistrar") + .setTargetId(domain.getDomainName()) .setEventTime(fakeClock.nowUtc()) .setBillingTime( fakeClock @@ -407,7 +405,7 @@ public class DomainBaseTest extends EntityTestCase { .setTransferStatus(TransferStatus.PENDING) .setTransferRequestTime(fakeClock.nowUtc().minusDays(4)) .setPendingTransferExpirationTime(fakeClock.nowUtc().plusDays(1)) - .setGainingRegistrarId("winner") + .setGainingRegistrarId("TheRegistrar") .setServerApproveBillingEvent(transferBillingEvent.createVKey()) .setServerApproveEntities(ImmutableSet.of(transferBillingEvent.createVKey())) .build()) @@ -418,8 +416,8 @@ public class DomainBaseTest extends EntityTestCase { GracePeriodStatus.ADD, domain.getRepoId(), fakeClock.nowUtc().plusDays(100), - "foo", - null)) + "TheRegistrar", + oneTimeBillKey)) .build(); DomainBase afterTransfer = domain.cloneProjectedAtTime(fakeClock.nowUtc().plusDays(1)); DateTime newExpirationTime = oldExpirationTime.plusYears(1); @@ -435,7 +433,7 @@ public class DomainBaseTest extends EntityTestCase { .nowUtc() .plusDays(1) .plus(Registry.get("com").getTransferGracePeriodLength()), - "winner", + "TheRegistrar", transferBillingEvent.createVKey(), afterTransfer.getGracePeriods().iterator().next().getGracePeriodId())); // If we project after the grace period expires all should be the same except the grace period. @@ -491,13 +489,13 @@ public class DomainBaseTest extends EntityTestCase { DomainBase beforeAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.minusDays(1)); assertThat(beforeAutoRenew.getLastEppUpdateTime()).isEqualTo(transferRequestDateTime); - assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("gaining"); + assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("TheRegistrar"); // If autorenew happens before transfer succeeds(before transfer grace period starts as well), // lastEppUpdateClientId should still be the current sponsor client id DomainBase afterAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.plusDays(1)); assertThat(afterAutoRenew.getLastEppUpdateTime()).isEqualTo(autorenewDateTime); - assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("losing"); + assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("NewRegistrar"); } @Test @@ -510,12 +508,12 @@ public class DomainBaseTest extends EntityTestCase { DomainBase beforeAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.minusDays(1)); assertThat(beforeAutoRenew.getLastEppUpdateTime()).isEqualTo(transferRequestDateTime); - assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("gaining"); + assertThat(beforeAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("TheRegistrar"); DomainBase afterTransferSuccess = domain.cloneProjectedAtTime(transferSuccessDateTime.plusDays(1)); assertThat(afterTransferSuccess.getLastEppUpdateTime()).isEqualTo(transferSuccessDateTime); - assertThat(afterTransferSuccess.getLastEppUpdateRegistrarId()).isEqualTo("gaining"); + assertThat(afterTransferSuccess.getLastEppUpdateRegistrarId()).isEqualTo("TheRegistrar"); } private void setupUnmodifiedDomain(DateTime oldExpirationTime) { @@ -542,7 +540,7 @@ public class DomainBaseTest extends EntityTestCase { DomainBase afterAutoRenew = domain.cloneProjectedAtTime(autorenewDateTime.plusDays(1)); assertThat(afterAutoRenew.getLastEppUpdateTime()).isEqualTo(autorenewDateTime); - assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("losing"); + assertThat(afterAutoRenew.getLastEppUpdateRegistrarId()).isEqualTo("NewRegistrar"); } @Test @@ -911,20 +909,8 @@ public class DomainBaseTest extends EntityTestCase { @Test void testContactFields() { - VKey contact3Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id3") - .setRepoId("4-COM") - .build()) - .createVKey(); - VKey contact4Key = - persistResource( - new ContactResource.Builder() - .setContactId("contact_id4") - .setRepoId("5-COM") - .build()) - .createVKey(); + VKey contact3Key = persistActiveContact("contact_id3").createVKey(); + VKey contact4Key = persistActiveContact("contact_id4").createVKey(); // Set all of the contacts. domain.setContactFields( diff --git a/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java b/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java index c7da11258..9b75c1a4f 100644 --- a/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java +++ b/core/src/test/java/google/registry/model/index/EppResourceIndexTest.java @@ -26,12 +26,19 @@ import com.google.common.collect.Iterables; import com.googlecode.objectify.Key; import google.registry.model.EntityTestCase; import google.registry.model.contact.ContactResource; +import google.registry.testing.TmOverrideExtension; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link EppResourceIndex}. */ class EppResourceIndexTest extends EntityTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private ContactResource contact; @BeforeEach diff --git a/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java b/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java index fd292f02d..9c3026d98 100644 --- a/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java +++ b/core/src/test/java/google/registry/model/ofy/CommitLogBucketTest.java @@ -27,13 +27,19 @@ import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.annotation.Cache; import google.registry.testing.AppEngineExtension; import google.registry.testing.InjectExtension; +import google.registry.testing.TmOverrideExtension; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link CommitLogBucket}. */ public class CommitLogBucketTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java b/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java index dfabb6c67..8766d3a6c 100644 --- a/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java +++ b/core/src/test/java/google/registry/model/ofy/DatastoreTransactionManagerTest.java @@ -27,14 +27,19 @@ import com.googlecode.objectify.annotation.Parent; import google.registry.model.ImmutableObject; import google.registry.model.annotations.InCrossTld; import google.registry.model.common.EntityGroupRoot; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link DatastoreTransactionManager}. */ public class DatastoreTransactionManagerTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder() @@ -54,7 +59,6 @@ public class DatastoreTransactionManagerTest { } @Entity - @EntityForTesting @InCrossTld private static class InCrossTldTestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java b/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java index 472f373c8..062153b54 100644 --- a/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java +++ b/core/src/test/java/google/registry/model/ofy/OfyCommitLogTest.java @@ -34,14 +34,20 @@ import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; import google.registry.testing.InjectExtension; import google.registry.testing.TestObject.TestVirtualObject; +import google.registry.testing.TmOverrideExtension; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests ensuring {@link Ofy} saves transactions to {@link CommitLogManifest}. */ public class OfyCommitLogTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder() diff --git a/core/src/test/java/google/registry/model/ofy/OfyTest.java b/core/src/test/java/google/registry/model/ofy/OfyTest.java index a2e2f6d3d..5d3456961 100644 --- a/core/src/test/java/google/registry/model/ofy/OfyTest.java +++ b/core/src/test/java/google/registry/model/ofy/OfyTest.java @@ -45,17 +45,17 @@ import google.registry.model.contact.ContactHistory; import google.registry.model.contact.ContactResource; import google.registry.model.domain.DomainBase; import google.registry.model.eppcommon.Trid; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.model.reporting.HistoryEntry; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import google.registry.testing.AppEngineExtension; import google.registry.testing.DatabaseHelper; import google.registry.testing.FakeClock; +import google.registry.testing.TmOverrideExtension; import google.registry.util.SystemClock; import java.util.ConcurrentModificationException; import java.util.function.Supplier; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -64,6 +64,10 @@ public class OfyTest { private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ")); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().withClock(fakeClock).build(); @@ -178,7 +182,6 @@ public class OfyTest { /** Simple entity class with lifecycle callbacks. */ @com.googlecode.objectify.annotation.Entity - @EntityForTesting public static class LifecycleObject extends ImmutableObject { @Parent Key parent = getCrossTldKey(); @@ -437,12 +440,4 @@ public class OfyTest { // Test the normal loading again to verify that we've restored the original session unchanged. assertThat(auditedOfy().load().entity(someObject).now()).isEqualTo(someObject.asHistoryEntry()); } - - @Test - void testReadOnly_failsWrite() { - Ofy ofy = new Ofy(fakeClock); - DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(fakeClock); - assertThrows(ReadOnlyModeException.class, () -> ofy.save().entity(someObject).now()); - DatabaseHelper.removeDatabaseMigrationSchedule(); - } } diff --git a/core/src/test/java/google/registry/model/replay/EntityTest.java b/core/src/test/java/google/registry/model/replay/EntityTest.java deleted file mode 100644 index ebb4022a8..000000000 --- a/core/src/test/java/google/registry/model/replay/EntityTest.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.model.replay; - -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; - -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import com.googlecode.objectify.annotation.Embed; -import com.googlecode.objectify.annotation.Parent; -import google.registry.model.ModelUtils; -import google.registry.model.common.GaeUserIdConverter; -import google.registry.persistence.VKey; -import google.registry.testing.DatastoreEntityExtension; -import io.github.classgraph.ClassGraph; -import io.github.classgraph.ClassInfo; -import io.github.classgraph.ClassInfoList; -import io.github.classgraph.ScanResult; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.lang.reflect.Method; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** - * Test to verify classes implement {@link SqlEntity} and {@link DatastoreEntity} when they should. - */ -public class EntityTest { - - @RegisterExtension - final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); - - private static final ImmutableSet> NON_CONVERTED_CLASSES = - ImmutableSet.of(GaeUserIdConverter.class); - - @Test - void testSqlEntityPersistence() { - try (ScanResult scanResult = scanForClasses()) { - // All javax.persistence entities must implement SqlEntity and vice versa - ImmutableSet javaxPersistenceClasses = - getAllClassesWithAnnotation(scanResult, javax.persistence.Entity.class.getName()); - ImmutableSet sqlEntityClasses = - getClassNames(scanResult.getClassesImplementing(SqlEntity.class.getName())); - assertThat(sqlEntityClasses).containsExactlyElementsIn(javaxPersistenceClasses); - - // All com.googlecode.objectify entities must implement DatastoreEntity and vice versa - ImmutableSet objectifyClasses = - getAllClassesWithAnnotation( - scanResult, com.googlecode.objectify.annotation.Entity.class.getName()); - ImmutableSet datastoreEntityClasses = - getClassNames(scanResult.getClassesImplementing(DatastoreEntity.class.getName())); - assertThat(datastoreEntityClasses).containsExactlyElementsIn(objectifyClasses); - } - } - - @Test - void testDatastoreEntityVKeyCreation() { - // For replication, we need to be able to convert from Key -> VKey for the relevant classes. - // This means that the relevant classes must have non-composite Objectify keys or must have a - // createVKey method - try (ScanResult scanResult = scanForClasses()) { - ImmutableSet> datastoreEntityClasses = - getClasses(scanResult.getClassesImplementing(DatastoreEntity.class.getName())); - // some classes aren't converted so they aren't relevant - ImmutableSet> vkeyConversionNecessaryClasses = - datastoreEntityClasses.stream() - .filter(clazz -> !DatastoreOnlyEntity.class.isAssignableFrom(clazz)) - .filter(clazz -> !NonReplicatedEntity.class.isAssignableFrom(clazz)) - .collect(toImmutableSet()); - - ImmutableSet.Builder> failedClasses = new ImmutableSet.Builder<>(); - for (Class clazz : vkeyConversionNecessaryClasses) { - if (hasKeyWithParent(clazz)) { - try { - Method createVKeyMethod = clazz.getMethod("createVKey", Key.class); - if (!createVKeyMethod.getReturnType().equals(VKey.class)) { - failedClasses.add(clazz); - } - } catch (NoSuchMethodException e) { - failedClasses.add(clazz); - } - } - } - assertWithMessage( - "Some DatastoreEntity classes with parents were missing createVKey methods: ") - .that(failedClasses.build()) - .isEmpty(); - } - } - - private boolean hasKeyWithParent(Class clazz) { - return ModelUtils.getAllFields(clazz).values().stream() - .anyMatch(field -> field.getAnnotation(Parent.class) != null); - } - - private ImmutableSet getAllClassesWithAnnotation( - ScanResult scanResult, String annotation) { - ImmutableSet.Builder result = new ImmutableSet.Builder<>(); - ClassInfoList classesWithAnnotation = scanResult.getClassesWithAnnotation(annotation); - result.addAll(getClassNames(classesWithAnnotation)); - classesWithAnnotation.stream() - .map(ClassInfo::getSubclasses) - .forEach(classInfoList -> result.addAll(getClassNames(classInfoList))); - return result.build(); - } - - private ImmutableSet> getClasses(ClassInfoList classInfoList) { - return classInfoList.stream() - .filter(ClassInfo::isStandardClass) - .map(ClassInfo::loadClass) - .filter( - clazz -> - !clazz.isAnnotationPresent(EntityForTesting.class) - && !clazz.isAnnotationPresent(Embed.class) - && !NON_CONVERTED_CLASSES.contains(clazz) - && !clazz.getName().contains("Test")) - .collect(toImmutableSet()); - } - - private ImmutableSet getClassNames(ClassInfoList classInfoList) { - return getClasses(classInfoList).stream().map(Class::getName).collect(toImmutableSet()); - } - - private ScanResult scanForClasses() { - return new ClassGraph() - .enableAnnotationInfo() - .ignoreClassVisibility() - .acceptPackages("google.registry") - .scan(); - } - - /** Entities that are solely used for testing, to avoid scanning them in {@link EntityTest}. */ - @Target(ElementType.TYPE) - @Retention(RetentionPolicy.RUNTIME) - public @interface EntityForTesting {} -} diff --git a/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java b/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java index 212d2b497..1dafe3bea 100644 --- a/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java +++ b/core/src/test/java/google/registry/model/translators/CommitLogRevisionsTranslatorFactoryTest.java @@ -27,13 +27,14 @@ import com.googlecode.objectify.annotation.Entity; import google.registry.model.common.CrossTldSingleton; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.Ofy; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; import google.registry.testing.InjectExtension; +import google.registry.testing.TmOverrideExtension; import java.util.List; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -43,11 +44,14 @@ public class CommitLogRevisionsTranslatorFactoryTest { private static final DateTime START_TIME = DateTime.parse("2000-01-01TZ"); @Entity(name = "ClrtfTestEntity") - @EntityForTesting public static class TestObject extends CrossTldSingleton { ImmutableSortedMap> revisions = ImmutableSortedMap.of(); } + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder() diff --git a/core/src/test/java/google/registry/persistence/BillingVKeyTest.java b/core/src/test/java/google/registry/persistence/BillingVKeyTest.java index 2b68e5c23..b32b7a9f3 100644 --- a/core/src/test/java/google/registry/persistence/BillingVKeyTest.java +++ b/core/src/test/java/google/registry/persistence/BillingVKeyTest.java @@ -26,7 +26,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.billing.BillingEvent; import google.registry.model.common.EntityGroupRoot; import google.registry.model.domain.DomainBase; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.model.reporting.HistoryEntry; import google.registry.persistence.BillingVKey.BillingEventVKey; import google.registry.persistence.BillingVKey.BillingRecurrenceVKey; @@ -80,7 +79,6 @@ class BillingVKeyTest { assertThat(persisted).isEqualTo(original); } - @EntityForTesting @Entity @javax.persistence.Entity private static class BillingVKeyTestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java b/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java index b888eea5a..26f2be5ca 100644 --- a/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java +++ b/core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java @@ -26,7 +26,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.common.EntityGroupRoot; import google.registry.model.domain.DomainBase; import google.registry.model.domain.DomainHistory.DomainHistoryId; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.AppEngineExtension; import google.registry.testing.DualDatabaseTest; @@ -82,7 +81,6 @@ class DomainHistoryVKeyTest { VKey.create(HistoryEntry.class, new DomainHistoryId("domainRepoId", 10L), ofyKey)); } - @EntityForTesting @Entity @javax.persistence.Entity(name = "TestEntity") private static class TestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java b/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java index 3dc744660..607f0fbe0 100644 --- a/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java +++ b/core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java @@ -22,7 +22,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import com.google.common.collect.ImmutableSet; import google.registry.model.ImmutableObject; -import google.registry.model.replay.NonReplicatedEntity; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import java.lang.reflect.Method; @@ -169,7 +168,7 @@ class EntityCallbacksListenerTest { } @Entity(name = "TestEntity") - private static class TestEntity extends ParentEntity implements NonReplicatedEntity { + private static class TestEntity extends ParentEntity { @Id String name = "id"; int nonTransientField = 0; diff --git a/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java b/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java index 0c1428452..5767e2124 100644 --- a/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/BloomFilterConverterTest.java @@ -22,7 +22,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import com.google.common.collect.ImmutableSet; import com.google.common.hash.BloomFilter; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import javax.persistence.Entity; @@ -49,7 +48,6 @@ class BloomFilterConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting public static class TestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java b/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java index 49ee2bfb0..9d01df530 100644 --- a/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/CurrencyUnitConverterTest.java @@ -20,7 +20,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import static org.junit.jupiter.api.Assertions.assertThrows; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import javax.persistence.Entity; @@ -77,7 +76,6 @@ public class CurrencyUnitConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting public static class TestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java b/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java index dc4391057..36b6c27a5 100644 --- a/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/DurationConverterTest.java @@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.insertInDb; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import javax.persistence.Entity; @@ -88,7 +87,6 @@ public class DurationConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting public static class DurationTestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java b/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java index c06209f7f..4889c527d 100644 --- a/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/InetAddressSetConverterTest.java @@ -21,7 +21,6 @@ import static google.registry.testing.DatabaseHelper.insertInDb; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.VKey; import google.registry.testing.AppEngineExtension; import java.net.InetAddress; @@ -73,7 +72,6 @@ public class InetAddressSetConverterTest { } @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. - @EntityForTesting private static class InetAddressSetTestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java b/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java index b7869634c..ca0bb818a 100644 --- a/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/JodaMoneyConverterTest.java @@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.common.collect.ImmutableMap; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import java.math.BigDecimal; @@ -289,7 +288,6 @@ public class JodaMoneyConverterTest { // Override entity name to exclude outer-class name in table name. Not necessary if class is not // inner class. @Entity(name = "TestEntity") - @EntityForTesting public static class TestEntity extends ImmutableObject { @Id String name = "id"; @@ -307,7 +305,6 @@ public class JodaMoneyConverterTest { // See comments on the annotation for TestEntity above for reason. @Entity(name = "ComplexTestEntity") - @EntityForTesting // This entity is used to test column override for embedded fields and collections. public static class ComplexTestEntity extends ImmutableObject { diff --git a/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java b/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java index 733a538e3..15a1e51ac 100644 --- a/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java +++ b/core/src/test/java/google/registry/persistence/converter/LocalDateConverterTest.java @@ -19,7 +19,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory. import static google.registry.testing.DatabaseHelper.insertInDb; import google.registry.model.ImmutableObject; -import google.registry.model.replay.EntityTest; import google.registry.persistence.VKey; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; @@ -63,7 +62,6 @@ public class LocalDateConverterTest { /** Override entity name to avoid the nested class reference. */ @Entity(name = "LocalDateConverterTestEntity") - @EntityTest.EntityForTesting private static class LocalDateConverterTestEntity extends ImmutableObject { @Id String name = "id"; diff --git a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java index 16f2a47ac..53dfbc2c2 100644 --- a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java +++ b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java @@ -27,7 +27,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import com.google.common.io.Resources; -import google.registry.model.common.DatabaseMigrationStateSchedule; import google.registry.persistence.HibernateSchemaExporter; import google.registry.persistence.NomulusPostgreSql; import google.registry.persistence.PersistenceModule; @@ -168,7 +167,7 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft if (!includeNomulusSchema) { File tempSqlFile = File.createTempFile("tempSqlFile", ".sql"); tempSqlFile.deleteOnExit(); - exporter.export(getTestEntities(), tempSqlFile); + exporter.export(extraEntityClasses, tempSqlFile); executeSql(new String(Files.readAllBytes(tempSqlFile.toPath()), StandardCharsets.UTF_8)); } assertReasonableNumDbConnections(); @@ -238,14 +237,16 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft ResultSet rs = statement.executeQuery( "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public';"); - ImmutableList.Builder tableNames = new ImmutableList.Builder<>(); + ImmutableList.Builder tableNamesBuilder = new ImmutableList.Builder<>(); while (rs.next()) { - tableNames.add('"' + rs.getString(1) + '"'); + tableNamesBuilder.add('"' + rs.getString(1) + '"'); + } + ImmutableList tableNames = tableNamesBuilder.build(); + if (!tableNames.isEmpty()) { + String sql = + String.format("TRUNCATE %s RESTART IDENTITY CASCADE", Joiner.on(',').join(tableNames)); + executeSql(sql); } - String sql = - String.format( - "TRUNCATE %s RESTART IDENTITY CASCADE", Joiner.on(',').join(tableNames.build())); - executeSql(sql); } catch (Exception e) { throw new RuntimeException(e); } @@ -344,17 +345,7 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft descriptor.getManagedClassNames().addAll(nonEntityClasses); } - getTestEntities().stream().map(Class::getName).forEach(descriptor::addClasses); + extraEntityClasses.stream().map(Class::getName).forEach(descriptor::addClasses); return Bootstrap.getEntityManagerFactoryBuilder(descriptor, properties).build(); } - - private ImmutableList> getTestEntities() { - // We have to add the DatabaseMigrationStateSchedule and TransactionEntity classes to extra - // entities, as they are required by the transaction manager factory and transaction replication - // mechanism, respectively. - return Stream.concat( - extraEntityClasses.stream(), - Stream.of(DatabaseMigrationStateSchedule.class, TransactionEntity.class)) - .collect(toImmutableList()); - } } diff --git a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java index dd04389a1..4dc11a2d1 100644 --- a/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java +++ b/core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java @@ -36,7 +36,6 @@ import google.registry.persistence.VKey; import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension; import google.registry.testing.DatabaseHelper; import google.registry.testing.FakeClock; -import google.registry.testing.TmOverrideExtension; import java.io.Serializable; import java.math.BigInteger; import java.sql.SQLException; @@ -49,7 +48,6 @@ import javax.persistence.IdClass; import javax.persistence.OptimisticLockException; import javax.persistence.RollbackException; import org.hibernate.exception.JDBCConnectionException; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -84,10 +82,6 @@ class JpaTransactionManagerImplTest { TestEntity.class, TestCompoundIdEntity.class, TestNamedCompoundIdEntity.class) .buildUnitTestExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - @Test void transact_succeeds() { assertPersonEmpty(); diff --git a/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java b/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java index cfbea25ae..2506c922d 100644 --- a/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java +++ b/core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java @@ -340,16 +340,6 @@ public class ReplicaSimulatingJpaTransactionManager implements JpaTransactionMan return delegate.isOfy(); } - @Override - public void putIgnoringReadOnlyWithoutBackup(Object entity) { - delegate.putIgnoringReadOnlyWithoutBackup(entity); - } - - @Override - public void deleteIgnoringReadOnlyWithoutBackup(VKey key) { - delegate.deleteIgnoringReadOnlyWithoutBackup(key); - } - @Override public void assertDelete(VKey key) { delegate.assertDelete(key); diff --git a/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java b/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java index f79c4fc91..a654e0852 100644 --- a/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java +++ b/core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java @@ -30,11 +30,8 @@ import com.googlecode.objectify.annotation.Id; import google.registry.model.ImmutableObject; import google.registry.model.ofy.DatastoreTransactionManager; import google.registry.model.ofy.Ofy; -import google.registry.model.replay.NonReplicatedEntity; import google.registry.persistence.VKey; -import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException; import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; import google.registry.testing.DualDatabaseTest; import google.registry.testing.FakeClock; import google.registry.testing.InjectExtension; @@ -409,13 +406,6 @@ public class TransactionManagerTest { assertThat(tm().transact(() -> tm().loadByKey(theEntity.key())).data).isEqualTo("foo"); } - @TestOfyAndSql - void testReadOnly_writeFails() { - DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(fakeClock); - assertThrows(ReadOnlyModeException.class, () -> tm().transact(() -> tm().put(theEntity))); - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - private static void assertEntityExists(TestEntity entity) { assertThat(tm().transact(() -> tm().exists(entity))).isTrue(); } @@ -449,7 +439,7 @@ public class TransactionManagerTest { @Entity(name = "TxnMgrTestEntity") @javax.persistence.Entity(name = "TestEntity") - private static class TestEntity extends TestEntityBase implements NonReplicatedEntity { + private static class TestEntity extends TestEntityBase { private String data; diff --git a/core/src/test/java/google/registry/persistence/transaction/TransactionTest.java b/core/src/test/java/google/registry/persistence/transaction/TransactionTest.java deleted file mode 100644 index b6e200442..000000000 --- a/core/src/test/java/google/registry/persistence/transaction/TransactionTest.java +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.persistence.transaction; - -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.ofy.ObjectifyService.auditedOfy; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.googlecode.objectify.Key; -import com.googlecode.objectify.annotation.Entity; -import com.googlecode.objectify.annotation.Id; -import google.registry.model.ImmutableObject; -import google.registry.model.ofy.CommitLogManifest; -import google.registry.model.ofy.CommitLogMutation; -import google.registry.model.ofy.Ofy; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.StreamCorruptedException; -import java.util.Comparator; -import java.util.List; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -class TransactionTest { - - private final FakeClock fakeClock = - new FakeClock(DateTime.parse("2000-01-01TZ")).setAutoIncrementByOneMilli(); - - @RegisterExtension - final AppEngineExtension appEngine = - AppEngineExtension.builder() - .withDatastoreAndCloudSql() - .withClock(fakeClock) - .withOfyTestEntities(TestEntity.class) - .withJpaUnitTestEntities(TestEntity.class) - .build(); - - @RegisterExtension public final InjectExtension inject = new InjectExtension(); - - private TestEntity fooEntity, barEntity; - - @BeforeEach - void beforeEach() { - inject.setStaticField(Ofy.class, "clock", fakeClock); - fooEntity = new TestEntity("foo"); - barEntity = new TestEntity("bar"); - } - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @Test - void testTransactionReplay() { - Transaction txn = new Transaction.Builder().addUpdate(fooEntity).addUpdate(barEntity).build(); - txn.writeToDatastore(); - - ofyTm() - .transact( - () -> { - assertThat(ofyTm().loadByKey(fooEntity.key())).isEqualTo(fooEntity); - assertThat(ofyTm().loadByKey(barEntity.key())).isEqualTo(barEntity); - }); - - txn = new Transaction.Builder().addDelete(barEntity.key()).build(); - txn.writeToDatastore(); - assertThat(ofyTm().exists(barEntity.key())).isEqualTo(false); - - assertThat( - auditedOfy().load().type(CommitLogMutation.class).list().stream() - .map(clm -> auditedOfy().load().fromEntity(clm.getEntity())) - .collect(toImmutableSet())) - .containsExactly(fooEntity, barEntity); - List manifests = auditedOfy().load().type(CommitLogManifest.class).list(); - manifests.sort(Comparator.comparing(CommitLogManifest::getCommitTime)); - assertThat(manifests.get(0).getDeletions()).isEmpty(); - assertThat(manifests.get(1).getDeletions()).containsExactly(Key.create(barEntity)); - } - - @Test - void testSerialization() throws Exception { - Transaction txn = new Transaction.Builder().addUpdate(barEntity).build(); - txn.writeToDatastore(); - - txn = new Transaction.Builder().addUpdate(fooEntity).addDelete(barEntity.key()).build(); - txn = Transaction.deserialize(txn.serialize()); - - txn.writeToDatastore(); - - ofyTm() - .transact( - () -> { - assertThat(ofyTm().loadByKey(fooEntity.key())).isEqualTo(fooEntity); - assertThat(ofyTm().exists(barEntity.key())).isEqualTo(false); - }); - } - - @Test - void testDeserializationErrors() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream out = new ObjectOutputStream(baos); - out.writeInt(12345); - out.close(); - assertThrows(IllegalArgumentException.class, () -> Transaction.deserialize(baos.toByteArray())); - - // Test with a short byte array. - assertThrows( - StreamCorruptedException.class, () -> Transaction.deserialize(new byte[] {1, 2, 3, 4})); - } - - @Test - void testTransactionSerialization() throws IOException { - DatabaseHelper.setMigrationScheduleToSqlPrimary(fakeClock); - jpaTm() - .transact( - () -> { - jpaTm().insert(fooEntity); - jpaTm().insert(barEntity); - }); - TransactionEntity txnEnt = - jpaTm().transact(() -> jpaTm().loadByKey(VKey.createSql(TransactionEntity.class, 1L))); - Transaction txn = Transaction.deserialize(txnEnt.getContents()); - txn.writeToDatastore(); - ofyTm() - .transact( - () -> { - assertThat(ofyTm().loadByKey(fooEntity.key())).isEqualTo(fooEntity); - assertThat(ofyTm().loadByKey(barEntity.key())).isEqualTo(barEntity); - }); - - // Verify that no transaction was persisted for the load transaction. - assertThat( - jpaTm().transact(() -> jpaTm().exists(VKey.createSql(TransactionEntity.class, 2L)))) - .isFalse(); - } - - @Test - void testTransactionSerializationDisabledByDefault() { - jpaTm() - .transact( - () -> { - jpaTm().insert(fooEntity); - jpaTm().insert(barEntity); - }); - assertThat(jpaTm().transact(() -> jpaTm().exists(VKey.createSql(TransactionEntity.class, 1L)))) - .isFalse(); - } - - @Entity(name = "TxnTestEntity") - @javax.persistence.Entity(name = "TestEntity") - private static class TestEntity extends ImmutableObject { - @Id @javax.persistence.Id private String name; - - private TestEntity() {} - - private TestEntity(String name) { - this.name = name; - } - - public VKey key() { - return VKey.create(TestEntity.class, name, Key.create(this)); - } - } -} diff --git a/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java b/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java index 011315c10..f99fdc87e 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java @@ -56,6 +56,7 @@ import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeLockHandler; import google.registry.testing.FakeResponse; import google.registry.testing.InjectExtension; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import google.registry.tldconfig.idn.IdnTableEnum; import google.registry.xjc.XjcXmlTransformer; @@ -84,12 +85,17 @@ import org.joda.time.DateTimeConstants; import org.joda.time.Duration; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link RdeStagingAction} in Datastore. */ public class RdeStagingActionDatastoreTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final BlobId XML_FILE = BlobId.of("rde-bucket", "lol_2000-01-01_full_S1_R0.xml.ghostryde"); private static final BlobId LENGTH_FILE = diff --git a/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java b/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java index 170e4c87c..685635b28 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingMapperTest.java @@ -27,10 +27,12 @@ import com.google.common.collect.ImmutableSetMultimap; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.Registrar.State; import google.registry.testing.AppEngineExtension; +import google.registry.testing.TmOverrideExtension; import google.registry.xml.ValidationMode; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.RegisterExtension; @@ -53,6 +55,10 @@ class RdeStagingMapperTest { private ArgumentCaptor depositFragmentCaptor = ArgumentCaptor.forClass(DepositFragment.class); + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension AppEngineExtension appEngineExtension = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java index 2361bba6f..dcc9cacd8 100644 --- a/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java +++ b/core/src/test/java/google/registry/rde/RdeStagingReducerTest.java @@ -43,6 +43,7 @@ import google.registry.testing.CloudTasksHelper; import google.registry.testing.CloudTasksHelper.TaskMatcher; import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeLockHandler; +import google.registry.testing.TmOverrideExtension; import google.registry.xml.ValidationMode; import java.io.IOException; import java.util.Iterator; @@ -52,12 +53,17 @@ import org.bouncycastle.openpgp.PGPPublicKey; import org.joda.time.DateTime; import org.joda.time.Duration; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link RdeStagingReducer}. */ class RdeStagingReducerTest { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension AppEngineExtension appEngineExtension = AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); diff --git a/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java b/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java index c82719111..a205c73de 100644 --- a/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java +++ b/core/src/test/java/google/registry/rde/RegistrarToXjcConverterTest.java @@ -21,13 +21,11 @@ import static google.registry.xjc.XjcXmlTransformer.marshalStrict; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableList; -import google.registry.model.ofy.Ofy; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.Registrar.State; import google.registry.model.registrar.RegistrarAddress; import google.registry.testing.AppEngineExtension; import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; import google.registry.xjc.rderegistrar.XjcRdeRegistrar; import google.registry.xjc.rderegistrar.XjcRdeRegistrarAddrType; import google.registry.xjc.rderegistrar.XjcRdeRegistrarPostalInfoEnumType; @@ -47,11 +45,11 @@ import org.junit.jupiter.api.extension.RegisterExtension; */ public class RegistrarToXjcConverterTest { + private final FakeClock clock = new FakeClock(DateTime.parse("2013-01-01T00:00:00Z")); + @RegisterExtension public final AppEngineExtension appEngine = - AppEngineExtension.builder().withDatastoreAndCloudSql().build(); - - @RegisterExtension public final InjectExtension inject = new InjectExtension(); + AppEngineExtension.builder().withDatastoreAndCloudSql().withClock(clock).build(); private Registrar registrar; @@ -86,9 +84,8 @@ public class RegistrarToXjcConverterTest { .setWhoisServer("whois.goblinmen.example") .setUrl("http://www.goblinmen.example") .build(); - FakeClock clock = new FakeClock(DateTime.parse("2013-01-01T00:00:00Z")); - inject.setStaticField(Ofy.class, "clock", clock); registrar = cloneAndSetAutoTimestamps(registrar); // Set the creation time in 2013. + registrar = registrar.asBuilder().setLastUpdateTime(null).build(); clock.setTo(DateTime.parse("2014-01-01T00:00:00Z")); registrar = cloneAndSetAutoTimestamps(registrar); // Set the update time in 2014. } diff --git a/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java b/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java index 3e6f47875..8107cd1c4 100644 --- a/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java +++ b/core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.when; import com.google.cloud.tasks.v2.HttpMethod; import com.google.common.net.MediaType; import google.registry.beam.BeamActionTestBase; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.reporting.ReportingModule; import google.registry.testing.AppEngineExtension; import google.registry.testing.CloudTasksHelper; @@ -62,7 +61,6 @@ class GenerateInvoicesActionTest extends BeamActionTestBase { "billing_bucket", "REG-INV", true, - PrimaryDatabase.DATASTORE, new YearMonth(2017, 10), emailUtils, cloudTasksUtils, @@ -97,7 +95,6 @@ class GenerateInvoicesActionTest extends BeamActionTestBase { "billing_bucket", "REG-INV", false, - PrimaryDatabase.DATASTORE, new YearMonth(2017, 10), emailUtils, cloudTasksUtils, @@ -122,7 +119,6 @@ class GenerateInvoicesActionTest extends BeamActionTestBase { "billing_bucket", "REG-INV", false, - PrimaryDatabase.DATASTORE, new YearMonth(2017, 10), emailUtils, cloudTasksUtils, diff --git a/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java b/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java index 48eddef0b..de59bec9e 100644 --- a/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java +++ b/core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java @@ -22,7 +22,6 @@ import static org.mockito.Mockito.when; import com.google.cloud.tasks.v2.HttpMethod; import com.google.common.net.MediaType; import google.registry.beam.BeamActionTestBase; -import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase; import google.registry.reporting.ReportingModule; import google.registry.testing.AppEngineExtension; import google.registry.testing.CloudTasksHelper; @@ -57,7 +56,6 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase { "gs://reporting-project/reporting-bucket/", "api_key/a", clock.nowUtc().toLocalDate(), - PrimaryDatabase.DATASTORE, true, clock, response, @@ -81,7 +79,6 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase { "gs://reporting-project/reporting-bucket/", "api_key/a", clock.nowUtc().toLocalDate(), - PrimaryDatabase.DATASTORE, true, clock, response, @@ -115,7 +112,6 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase { "gs://reporting-project/reporting-bucket/", "api_key/a", clock.nowUtc().toLocalDate(), - PrimaryDatabase.DATASTORE, false, clock, response, diff --git a/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java b/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java index b90996d4f..bf7a15752 100644 --- a/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java +++ b/core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java @@ -27,7 +27,6 @@ import google.registry.model.registrar.RegistrarContact; import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension; import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.TmOverrideExtension; import google.registry.util.SerializeUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; @@ -45,10 +44,6 @@ class RegistrarContactTest { JpaIntegrationWithCoverageExtension jpa = new JpaTestExtensions.Builder().buildIntegrationWithCoverageExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private Registrar testRegistrar; private RegistrarContact testRegistrarPoc; diff --git a/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java b/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java index 8a559359d..6ab068d7a 100644 --- a/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java +++ b/core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java @@ -30,7 +30,6 @@ import google.registry.persistence.transaction.JpaTestExtensions; import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension; import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; -import google.registry.testing.TmOverrideExtension; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; @@ -50,10 +49,6 @@ public class RegistrarDaoTest { JpaIntegrationWithCoverageExtension jpa = new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension(); - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - private final VKey registrarKey = VKey.createSql(Registrar.class, "registrarId"); private Registrar testRegistrar; diff --git a/core/src/test/java/google/registry/schema/replay/SqlEntityTest.java b/core/src/test/java/google/registry/schema/replay/SqlEntityTest.java deleted file mode 100644 index 09502c332..000000000 --- a/core/src/test/java/google/registry/schema/replay/SqlEntityTest.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.schema.replay; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; - -import google.registry.model.registrar.Registrar; -import google.registry.model.registrar.RegistrarContact; -import google.registry.model.registrar.RegistrarContact.RegistrarPocId; -import google.registry.persistence.VKey; -import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatastoreEntityExtension; -import google.registry.testing.TmOverrideExtension; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Unit tests for {@link SqlEntity#getPrimaryKeyString}. */ -public class SqlEntityTest { - - @RegisterExtension - @Order(1) - final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); - - @RegisterExtension - final AppEngineExtension database = - new AppEngineExtension.Builder().withCloudSql().withoutCannedData().build(); - - @RegisterExtension - @Order(Order.DEFAULT + 1) - TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa(); - - @BeforeEach - void setup() throws Exception { - AppEngineExtension.loadInitialData(); - } - - @Test - void getPrimaryKeyString_oneIdColumn() { - // AppEngineExtension canned data: Registrar1 - assertThat( - tm().transact(() -> tm().loadByKey(Registrar.createVKey("NewRegistrar"))) - .getPrimaryKeyString()) - .contains("NewRegistrar"); - } - - @Test - void getPrimaryKeyString_multiId() { - // AppEngineExtension canned data: RegistrarContact1 - VKey key = - VKey.createSql( - RegistrarContact.class, new RegistrarPocId("janedoe@theregistrar.com", "NewRegistrar")); - String expected = "emailAddress=janedoe@theregistrar.com\n registrarId=NewRegistrar"; - assertThat(tm().transact(() -> tm().loadByKey(key)).getPrimaryKeyString()).contains(expected); - } -} diff --git a/core/src/test/java/google/registry/testing/DatabaseHelper.java b/core/src/test/java/google/registry/testing/DatabaseHelper.java index 88a0dec37..c4fa20414 100644 --- a/core/src/test/java/google/registry/testing/DatabaseHelper.java +++ b/core/src/test/java/google/registry/testing/DatabaseHelper.java @@ -71,8 +71,6 @@ import google.registry.model.ImmutableObject; import google.registry.model.billing.BillingEvent; import google.registry.model.billing.BillingEvent.Flag; import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; import google.registry.model.contact.ContactAuthInfo; import google.registry.model.contact.ContactHistory; import google.registry.model.contact.ContactResource; @@ -126,7 +124,6 @@ import org.joda.money.CurrencyUnit; import org.joda.money.Money; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.joda.time.Duration; /** Static utils for setting up test resources. */ public class DatabaseHelper { @@ -1210,7 +1207,7 @@ public class DatabaseHelper { * entities. */ public static void insertSimpleResources(final Iterable resources) { - tm().transact(() -> tm().insertAllWithoutBackup(ImmutableList.copyOf(resources))); + tm().transact(() -> tm().putAllWithoutBackup(ImmutableList.copyOf(resources))); maybeAdvanceClock(); // Force the session to be cleared so that when we read it back, we read from Datastore // and not from the transaction's session cache. @@ -1226,18 +1223,10 @@ public class DatabaseHelper { /** Force the create and update timestamps to get written into the resource. */ public static R cloneAndSetAutoTimestamps(final R resource) { - R result; - if (tm().isOfy()) { - result = - tm().transact( - () -> auditedOfy().load().fromEntity(auditedOfy().save().toEntity(resource))); - } else { - // We have to separate the read and write operation into different transactions - // otherwise JPA would just return the input entity instead of actually creating a - // clone. - tm().transact(() -> tm().put(resource)); - result = tm().transact(() -> tm().loadByEntity(resource)); - } + // We have to separate the read and write operation into different transactions otherwise JPA + // would just return the input entity instead of actually creating a clone. + tm().transact(() -> tm().put(resource)); + R result = tm().transact(() -> tm().loadByEntity(resource)); maybeAdvanceClock(); return result; } @@ -1408,104 +1397,5 @@ public class DatabaseHelper { return entity; } - /** - * Sets a DATASTORE_PRIMARY_NO_ASYNC state on the {@link DatabaseMigrationStateSchedule}. - * - *

In order to allow for tests to manipulate the clock how they need, we start the transitions - * one millisecond after the clock's current time (in case the clock's current value is - * START_OF_TIME). We then advance the clock one second so that we're in the - * DATASTORE_PRIMARY_READ_ONLY phase. - * - *

We must use the current time, otherwise the setting of the migration state will fail due to - * an invalid transition. - */ - public static void setMigrationScheduleToDatastorePrimaryNoAsync(FakeClock fakeClock) { - DateTime now = fakeClock.nowUtc(); - jpaTm() - .transact( - () -> - DatabaseMigrationStateSchedule.set( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMillis(1), - MigrationState.DATASTORE_PRIMARY, - now.plusMillis(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC))); - fakeClock.advanceBy(Duration.standardSeconds(1)); - } - - /** - * Sets a DATASTORE_PRIMARY_READ_ONLY state on the {@link DatabaseMigrationStateSchedule}. - * - *

In order to allow for tests to manipulate the clock how they need, we start the transitions - * one millisecond after the clock's current time (in case the clock's current value is - * START_OF_TIME). We then advance the clock one second so that we're in the - * DATASTORE_PRIMARY_READ_ONLY phase. - * - *

We must use the current time, otherwise the setting of the migration state will fail due to - * an invalid transition. - */ - public static void setMigrationScheduleToDatastorePrimaryReadOnly(FakeClock fakeClock) { - DateTime now = fakeClock.nowUtc(); - jpaTm() - .transact( - () -> - DatabaseMigrationStateSchedule.set( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMillis(1), - MigrationState.DATASTORE_PRIMARY, - now.plusMillis(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusMillis(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY))); - fakeClock.advanceBy(Duration.standardSeconds(1)); - } - - /** - * Sets a SQL_PRIMARY state on the {@link DatabaseMigrationStateSchedule}. - * - *

In order to allow for tests to manipulate the clock how they need, we start the transitions - * one millisecond after the clock's current time (in case the clock's current value is - * START_OF_TIME). We then advance the clock one second so that we're in the SQL_PRIMARY phase. - * - *

We must use the current time, otherwise the setting of the migration state will fail due to - * an invalid transition. - */ - public static void setMigrationScheduleToSqlPrimary(FakeClock fakeClock) { - DateTime now = fakeClock.nowUtc(); - jpaTm() - .transact( - () -> - DatabaseMigrationStateSchedule.set( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMillis(1), - MigrationState.DATASTORE_PRIMARY, - now.plusMillis(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusMillis(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - now.plusMillis(4), - MigrationState.SQL_PRIMARY))); - fakeClock.advanceBy(Duration.standardSeconds(1)); - } - - /** Removes the database migration schedule, in essence transitioning to DATASTORE_ONLY. */ - public static void removeDatabaseMigrationSchedule() { - // use the raw calls because going SQL_PRIMARY -> DATASTORE_ONLY is not valid - jpaTm() - .transact( - () -> - jpaTm() - .putIgnoringReadOnlyWithoutBackup( - new DatabaseMigrationStateSchedule( - DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP))); - DatabaseMigrationStateSchedule.CACHE.invalidateAll(); - } - private DatabaseHelper() {} } diff --git a/core/src/test/java/google/registry/testing/TestObject.java b/core/src/test/java/google/registry/testing/TestObject.java index d54984164..7491a8a26 100644 --- a/core/src/test/java/google/registry/testing/TestObject.java +++ b/core/src/test/java/google/registry/testing/TestObject.java @@ -23,16 +23,13 @@ import com.googlecode.objectify.annotation.Parent; import google.registry.model.ImmutableObject; import google.registry.model.annotations.VirtualEntity; import google.registry.model.common.EntityGroupRoot; -import google.registry.model.replay.DatastoreAndSqlEntity; -import google.registry.model.replay.EntityTest.EntityForTesting; import google.registry.persistence.VKey; import javax.persistence.Transient; /** A test model object that can be persisted in any entity group. */ @Entity @javax.persistence.Entity -@EntityForTesting -public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity { +public class TestObject extends ImmutableObject { @Parent @Transient Key parent; @@ -75,7 +72,6 @@ public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity /** A test @VirtualEntity model object, which should not be persisted. */ @Entity @VirtualEntity - @EntityForTesting public static class TestVirtualObject extends ImmutableObject { @Id String id; diff --git a/core/src/test/java/google/registry/testing/TmOverrideExtension.java b/core/src/test/java/google/registry/testing/TmOverrideExtension.java index 247285423..6c6d658df 100644 --- a/core/src/test/java/google/registry/testing/TmOverrideExtension.java +++ b/core/src/test/java/google/registry/testing/TmOverrideExtension.java @@ -14,9 +14,9 @@ package google.registry.testing; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.persistence.transaction.TransactionManager; import google.registry.persistence.transaction.TransactionManagerFactory; import org.junit.jupiter.api.extension.AfterEachCallback; @@ -35,35 +35,17 @@ import org.junit.jupiter.api.extension.ExtensionContext; *

This extension is incompatible with {@link DualDatabaseTest}. Use either that or this, but not * both. */ +@DeleteAfterMigration public final class TmOverrideExtension implements BeforeEachCallback, AfterEachCallback { - private static enum TmOverride { - OFY, - JPA; - } - - private final TmOverride tmOverride; - - private TmOverrideExtension(TmOverride tmOverride) { - this.tmOverride = tmOverride; - } - /** Use the {@link google.registry.model.ofy.DatastoreTransactionManager} for all tests. */ public static TmOverrideExtension withOfy() { - return new TmOverrideExtension(TmOverride.OFY); - } - - /** - * Use the {@link google.registry.persistence.transaction.JpaTransactionManager} for all tests. - */ - public static TmOverrideExtension withJpa() { - return new TmOverrideExtension(TmOverride.JPA); + return new TmOverrideExtension(); } @Override public void beforeEach(ExtensionContext context) { - TransactionManagerFactory.setTmOverrideForTest( - tmOverride == TmOverride.OFY ? ofyTm() : jpaTm()); + TransactionManagerFactory.setTmOverrideForTest(ofyTm()); } @Override diff --git a/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java b/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java index 6a137da2f..4d3148e48 100644 --- a/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java +++ b/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java @@ -89,13 +89,18 @@ class NordnUploadActionTest { private static final String LOCATION_URL = "http://trololol"; + private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z")); + @RegisterExtension public final AppEngineExtension appEngine = - AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build(); + AppEngineExtension.builder() + .withDatastoreAndCloudSql() + .withClock(clock) + .withTaskQueue() + .build(); @RegisterExtension public final InjectExtension inject = new InjectExtension(); - - private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z")); + private final LordnRequestInitializer lordnRequestInitializer = new LordnRequestInitializer(Optional.of("attack")); private final NordnUploadAction action = new NordnUploadAction(); diff --git a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java index dbbc7f043..5a8f0840e 100644 --- a/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java +++ b/core/src/test/java/google/registry/tools/CompareDbBackupsTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.io.Resources; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.testing.DatastoreEntityExtension; import google.registry.tools.EntityWrapper.Property; import java.io.ByteArrayOutputStream; @@ -32,6 +33,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; +@DeleteAfterMigration public class CompareDbBackupsTest { private static final int BASE_ID = 1001; diff --git a/core/src/test/java/google/registry/tools/CreateTldCommandTest.java b/core/src/test/java/google/registry/tools/CreateTldCommandTest.java index 60deb80f4..bcbc05fdf 100644 --- a/core/src/test/java/google/registry/tools/CreateTldCommandTest.java +++ b/core/src/test/java/google/registry/tools/CreateTldCommandTest.java @@ -55,9 +55,9 @@ class CreateTldCommandTest extends CommandTestCase { @Test void testSuccess() throws Exception { - DateTime before = DateTime.now(UTC); + DateTime before = fakeClock.nowUtc(); runCommandForced("xn--q9jyb4c", "--roid_suffix=Q9JYB4C", "--dns_writers=FooDnsWriter"); - DateTime after = DateTime.now(UTC); + DateTime after = fakeClock.nowUtc(); Registry registry = Registry.get("xn--q9jyb4c"); assertThat(registry).isNotNull(); diff --git a/core/src/test/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommandTest.java b/core/src/test/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommandTest.java deleted file mode 100644 index 40c433c2d..000000000 --- a/core/src/test/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommandTest.java +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2020 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.testing.DatabaseHelper.createTld; -import static google.registry.testing.DatabaseHelper.loadAllOf; -import static google.registry.testing.DatabaseHelper.persistActiveDomain; -import static google.registry.testing.DatabaseHelper.persistResource; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.joda.money.CurrencyUnit.USD; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableSet; -import com.googlecode.objectify.Key; -import google.registry.model.billing.BillingEvent; -import google.registry.model.billing.BillingEvent.Reason; -import google.registry.model.domain.DomainBase; -import google.registry.model.domain.DomainHistory; -import google.registry.model.domain.Period; -import google.registry.model.eppcommon.Trid; -import google.registry.model.poll.PollMessage; -import google.registry.model.reporting.HistoryEntry; -import google.registry.model.transfer.DomainTransferData; -import org.joda.money.Money; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -/** - * Unit tests for {@link DedupeOneTimeBillingEventIdsCommand}. - * - *

Note that these are _not_ dual database tests even though the action has been converted. The - * dedupe was strictly a one-time event that needed to be done prior to moving to SQL. It should no - * longer be necessary and we may want to simply remove the command. - */ -class DedupeOneTimeBillingEventIdsCommandTest - extends CommandTestCase { - - DomainBase domain; - DomainHistory historyEntry; - PollMessage.Autorenew autorenewToResave; - BillingEvent.OneTime billingEventToResave; - - @BeforeEach - void beforeEach() { - createTld("foobar"); - domain = persistActiveDomain("foo.foobar"); - historyEntry = persistHistoryEntry(domain); - autorenewToResave = persistAutorenewPollMessage(historyEntry); - billingEventToResave = persistBillingEvent(historyEntry); - } - - @Test - void resaveBillingEvent_succeeds() throws Exception { - runCommand( - "--force", - "--key_paths_file", - writeToNamedTmpFile("keypath.txt", getKeyPathLiteral(billingEventToResave))); - - int count = 0; - for (BillingEvent.OneTime billingEvent : loadAllOf(BillingEvent.OneTime.class)) { - count++; - assertThat(billingEvent.getId()).isNotEqualTo(billingEventToResave.getId()); - assertThat(billingEvent.asBuilder().setId(billingEventToResave.getId()).build()) - .isEqualTo(billingEventToResave); - } - assertThat(count).isEqualTo(1); - } - - @Test - void resaveBillingEvent_failsWhenReferredByDomain() { - persistResource( - domain - .asBuilder() - .setTransferData( - new DomainTransferData.Builder() - .setServerApproveEntities(ImmutableSet.of(billingEventToResave.createVKey())) - .build()) - .build()); - - assertThrows( - IllegalStateException.class, - () -> - runCommand( - "--force", - "--key_paths_file", - writeToNamedTmpFile("keypath.txt", getKeyPathLiteral(billingEventToResave)))); - } - - private PollMessage.Autorenew persistAutorenewPollMessage(HistoryEntry historyEntry) { - return persistResource( - new PollMessage.Autorenew.Builder() - .setRegistrarId("TheRegistrar") - .setEventTime(fakeClock.nowUtc()) - .setMsg("Test poll message") - .setParent(historyEntry) - .setAutorenewEndTime(fakeClock.nowUtc().plusDays(365)) - .setTargetId("foobar.foo") - .build()); - } - - private BillingEvent.OneTime persistBillingEvent(DomainHistory historyEntry) { - return persistResource( - new BillingEvent.OneTime.Builder() - .setRegistrarId("a registrar") - .setTargetId("foo.tld") - .setParent(historyEntry) - .setReason(Reason.CREATE) - .setFlags(ImmutableSet.of(BillingEvent.Flag.ANCHOR_TENANT)) - .setPeriodYears(2) - .setCost(Money.of(USD, 1)) - .setEventTime(fakeClock.nowUtc()) - .setBillingTime(fakeClock.nowUtc().plusDays(5)) - .build()); - } - - private DomainHistory persistHistoryEntry(DomainBase parent) { - return persistResource( - new DomainHistory.Builder() - .setDomain(parent) - .setType(HistoryEntry.Type.DOMAIN_CREATE) - .setPeriod(Period.create(1, Period.Unit.YEARS)) - .setXmlBytes("".getBytes(UTF_8)) - .setModificationTime(fakeClock.nowUtc()) - .setRegistrarId("foo") - .setTrid(Trid.create("ABC-123", "server-trid")) - .setBySuperuser(false) - .setReason("reason") - .setRequestedByRegistrar(false) - .build()); - } - - private static String getKeyPathLiteral(Object entity) { - Key key = Key.create(entity); - return String.format( - "\"DomainBase\", \"%s\", \"HistoryEntry\", %s, \"%s\", %s", - key.getParent().getParent().getName(), key.getParent().getId(), key.getKind(), key.getId()); - } -} diff --git a/core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java b/core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java deleted file mode 100644 index 7b3a910bc..000000000 --- a/core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.util.DateTimeUtils.START_OF_TIME; - -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.DualDatabaseTest; -import google.registry.testing.TestOfyAndSql; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; - -/** Tests for {@link GetDatabaseMigrationStateCommand}. */ -@DualDatabaseTest -public class GetDatabaseMigrationStateCommandTest - extends CommandTestCase { - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @TestOfyAndSql - void testInitial_returnsDatastoreOnly() throws Exception { - runCommand(); - assertStdoutIs( - String.format("Current migration schedule: %s\n", DEFAULT_TRANSITION_MAP.toValueMap())); - } - - @TestOfyAndSql - void testFullSchedule() throws Exception { - DateTime now = fakeClock.nowUtc(); - ImmutableSortedMap transitions = - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusHours(1), - MigrationState.DATASTORE_PRIMARY, - now.plusHours(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusHours(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - now.plusHours(4), - MigrationState.SQL_PRIMARY, - now.plusHours(5), - MigrationState.SQL_ONLY); - jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions)); - runCommand(); - assertStdoutIs(String.format("Current migration schedule: %s\n", transitions)); - } -} diff --git a/core/src/test/java/google/registry/tools/GetDomainCommandTest.java b/core/src/test/java/google/registry/tools/GetDomainCommandTest.java index 3c5c15b24..c64e41575 100644 --- a/core/src/test/java/google/registry/tools/GetDomainCommandTest.java +++ b/core/src/test/java/google/registry/tools/GetDomainCommandTest.java @@ -19,19 +19,15 @@ import static google.registry.testing.DatabaseHelper.newDomainBase; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistDeletedDomain; import static google.registry.testing.DatabaseHelper.persistResource; -import static org.joda.time.DateTimeZone.UTC; import static org.junit.jupiter.api.Assertions.assertThrows; import com.beust.jcommander.ParameterException; -import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** Unit tests for {@link GetDomainCommand}. */ class GetDomainCommandTest extends CommandTestCase { - private DateTime now = DateTime.now(UTC); - @BeforeEach void beforeEach() { createTld("tld"); @@ -55,7 +51,7 @@ class GetDomainCommandTest extends CommandTestCase { persistActiveDomain("example.tld"); runCommand("example.tld", "--expand"); assertInStdout("fullyQualifiedDomainName=example.tld"); - assertInStdout("contactId=contact1234"); + assertInStdout("contact=Key(ContactResource(\"3-ROID\"))"); assertInStdout( "Websafe key: " + "kind:DomainBase" @@ -70,7 +66,7 @@ class GetDomainCommandTest extends CommandTestCase { persistActiveDomain("xn--aualito-txac.xn--q9jyb4c"); runCommand("çauçalito.みんな", "--expand"); assertInStdout("fullyQualifiedDomainName=xn--aualito-txac.xn--q9jyb4c"); - assertInStdout("contactId=contact1234"); + assertInStdout("contact=Key(ContactResource(\"4-ROID\"))"); } @Test @@ -94,15 +90,18 @@ class GetDomainCommandTest extends CommandTestCase { @Test void testSuccess_domainDeletedInFuture() throws Exception { - persistResource(newDomainBase("example.tld").asBuilder() - .setDeletionTime(now.plusDays(1)).build()); - runCommand("example.tld", "--read_timestamp=" + now.plusMonths(1)); + persistResource( + newDomainBase("example.tld") + .asBuilder() + .setDeletionTime(fakeClock.nowUtc().plusDays(1)) + .build()); + runCommand("example.tld", "--read_timestamp=" + fakeClock.nowUtc().plusMonths(1)); assertInStdout("Domain 'example.tld' does not exist or is deleted"); } @Test void testSuccess_deletedDomain() throws Exception { - persistDeletedDomain("example.tld", now.minusDays(1)); + persistDeletedDomain("example.tld", fakeClock.nowUtc().minusDays(1)); runCommand("example.tld"); assertInStdout("Domain 'example.tld' does not exist or is deleted"); } diff --git a/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java b/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java index fa0ac93ac..0782960a0 100644 --- a/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java +++ b/core/src/test/java/google/registry/tools/GetResourceByKeyCommandTest.java @@ -26,13 +26,20 @@ import static org.joda.time.DateTimeZone.UTC; import static org.junit.jupiter.api.Assertions.assertThrows; import com.beust.jcommander.ParameterException; +import google.registry.testing.TmOverrideExtension; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link GetResourceByKeyCommand}. */ class GetResourceByKeyCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private DateTime now = DateTime.now(UTC); @BeforeEach diff --git a/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java b/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java index f2426de0c..07c994991 100644 --- a/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java +++ b/core/src/test/java/google/registry/tools/LevelDbFileBuilder.java @@ -20,6 +20,7 @@ import static google.registry.tools.LevelDbLogReader.HEADER_SIZE; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.EntityTranslator; import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; +import google.registry.model.annotations.DeleteAfterMigration; import google.registry.tools.LevelDbLogReader.ChunkType; import java.io.File; import java.io.FileNotFoundException; @@ -27,6 +28,7 @@ import java.io.FileOutputStream; import java.io.IOException; /** Utility class for building a leveldb logfile. */ +@DeleteAfterMigration public final class LevelDbFileBuilder { private final FileOutputStream out; diff --git a/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java b/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java index 04b5ed686..b66261f43 100644 --- a/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java +++ b/core/src/test/java/google/registry/tools/LevelDbFileBuilderTest.java @@ -25,10 +25,12 @@ import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; import google.registry.model.contact.ContactResource; import google.registry.testing.AppEngineExtension; import google.registry.testing.DatabaseHelper; +import google.registry.testing.TmOverrideExtension; import google.registry.tools.EntityWrapper.Property; import java.io.File; import java.io.IOException; import java.nio.file.Path; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; @@ -40,6 +42,10 @@ public class LevelDbFileBuilderTest { @TempDir Path tmpDir; + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @RegisterExtension public final AppEngineExtension appEngine = AppEngineExtension.builder().withDatastoreAndCloudSql().build(); diff --git a/core/src/test/java/google/registry/tools/MutatingCommandTest.java b/core/src/test/java/google/registry/tools/MutatingCommandTest.java index 285d49d55..4417a216a 100644 --- a/core/src/test/java/google/registry/tools/MutatingCommandTest.java +++ b/core/src/test/java/google/registry/tools/MutatingCommandTest.java @@ -19,6 +19,7 @@ import static com.google.common.truth.Truth.assertThat; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.deleteResource; +import static google.registry.testing.DatabaseHelper.existsInDb; import static google.registry.testing.DatabaseHelper.loadByEntity; import static google.registry.testing.DatabaseHelper.persistActiveHost; import static google.registry.testing.DatabaseHelper.persistNewRegistrar; @@ -181,10 +182,10 @@ public class MutatingCommandTest { + registrar2 + "\n"); String results = command.execute(); assertThat(results).isEqualTo("Updated 4 entities.\n"); - assertThat(loadByEntity(host1)).isNull(); - assertThat(loadByEntity(host2)).isNull(); - assertThat(loadByEntity(registrar1)).isNull(); - assertThat(loadByEntity(registrar2)).isNull(); + assertThat(existsInDb(host1)).isFalse(); + assertThat(existsInDb(host2)).isFalse(); + assertThat(existsInDb(registrar1)).isFalse(); + assertThat(existsInDb(registrar2)).isFalse(); } @Test @@ -241,9 +242,9 @@ public class MutatingCommandTest { + "blockPremiumNames: false -> true\n"); String results = command.execute(); assertThat(results).isEqualTo("Updated 4 entities.\n"); - assertThat(loadByEntity(host1)).isNull(); + assertThat(existsInDb(host1)).isFalse(); assertThat(loadByEntity(host2)).isEqualTo(newHost2); - assertThat(loadByEntity(registrar1)).isNull(); + assertThat(existsInDb(registrar1)).isFalse(); assertThat(loadByEntity(registrar2)).isEqualTo(newRegistrar2); } @@ -282,7 +283,7 @@ public class MutatingCommandTest { IllegalStateException thrown = assertThrows(IllegalStateException.class, command::execute); assertThat(thrown).hasMessageThat().contains("Entity changed since init() was called."); - assertThat(loadByEntity(host1)).isNull(); + assertThat(existsInDb(host1)).isFalse(); assertThat(loadByEntity(host2)).isEqualTo(newHost2); // These two shouldn't've changed. assertThat(loadByEntity(registrar1)).isEqualTo(registrar1); diff --git a/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java b/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java index f9dd48551..ba5f1868c 100644 --- a/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java +++ b/core/src/test/java/google/registry/tools/RegistrarContactCommandTest.java @@ -23,6 +23,7 @@ import static google.registry.testing.DatabaseHelper.loadRegistrar; import static google.registry.testing.DatabaseHelper.persistResource; import static google.registry.testing.DatabaseHelper.persistSimpleResource; import static google.registry.testing.DatabaseHelper.persistSimpleResources; +import static google.registry.testing.DatabaseHelper.putInDb; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -334,8 +335,7 @@ class RegistrarContactCommandTest extends CommandTestCase { + + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + DomainBase domain; HistoryEntry historyEntry; diff --git a/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java b/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java index 77e037e7b..29de8082f 100644 --- a/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java +++ b/core/src/test/java/google/registry/tools/RenewDomainCommandTest.java @@ -25,29 +25,20 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.beust.jcommander.ParameterException; import com.google.common.collect.ImmutableMap; import google.registry.model.domain.DomainBase; -import google.registry.model.ofy.Ofy; import google.registry.model.registrar.Registrar; -import google.registry.testing.FakeClock; -import google.registry.testing.InjectExtension; -import google.registry.util.Clock; import java.util.List; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.shaded.com.google.common.collect.ImmutableList; /** Unit tests for {@link RenewDomainCommand}. */ public class RenewDomainCommandTest extends EppToolCommandTestCase { - @RegisterExtension public final InjectExtension inject = new InjectExtension(); - - private final Clock clock = new FakeClock(DateTime.parse("2015-04-05T05:05:05Z")); - @BeforeEach void beforeEach() { - inject.setStaticField(Ofy.class, "clock", clock); - command.clock = clock; + fakeClock.setTo(DateTime.parse("2015-04-05T05:05:05Z")); + command.clock = fakeClock; } @Test diff --git a/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java b/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java index 0e572eddf..6e86bb659 100644 --- a/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java +++ b/core/src/test/java/google/registry/tools/ResaveEntitiesCommandTest.java @@ -25,11 +25,18 @@ import google.registry.model.ImmutableObject; import google.registry.model.contact.ContactResource; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveEntitiesCommand}. */ class ResaveEntitiesCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @Test void testSuccess_createsCommitLogs() throws Exception { ContactResource contact1 = persistActiveContact("contact1"); diff --git a/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java b/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java index 156f3881b..a76e5e607 100644 --- a/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java +++ b/core/src/test/java/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java @@ -28,12 +28,19 @@ import google.registry.model.ofy.CommitLogMutation; import google.registry.model.registrar.Registrar; import google.registry.model.registrar.RegistrarContact; import google.registry.model.tld.Registry; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveEnvironmentEntitiesCommand}. */ class ResaveEnvironmentEntitiesCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @Test void testSuccess_noop() throws Exception { // Get rid of all the entities that this command runs on so that it does nothing. diff --git a/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java b/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java index 35fa8ddcc..0cc47d5f9 100644 --- a/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java +++ b/core/src/test/java/google/registry/tools/ResaveEppResourcesCommandTest.java @@ -22,11 +22,18 @@ import google.registry.model.ImmutableObject; import google.registry.model.contact.ContactResource; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; +import google.registry.testing.TmOverrideExtension; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveEppResourceCommand}. */ class ResaveEppResourcesCommandTest extends CommandTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + @Test void testSuccess_createsCommitLogs() throws Exception { ContactResource contact = persistActiveContact("contact"); diff --git a/core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java b/core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java deleted file mode 100644 index 12f75ec16..000000000 --- a/core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2021 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.model.common.DatabaseMigrationStateSchedule.DEFAULT_TRANSITION_MAP; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; -import static google.registry.util.DateTimeUtils.START_OF_TIME; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.beust.jcommander.ParameterException; -import com.google.common.collect.ImmutableSortedMap; -import google.registry.model.common.DatabaseMigrationStateSchedule; -import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.DualDatabaseTest; -import google.registry.testing.TestOfyAndSql; -import org.joda.time.DateTime; -import org.junit.jupiter.api.AfterEach; - -/** Tests for {@link SetDatabaseMigrationStateCommand}. */ -@DualDatabaseTest -public class SetDatabaseMigrationStateCommandTest - extends CommandTestCase { - - @AfterEach - void afterEach() { - DatabaseHelper.removeDatabaseMigrationSchedule(); - } - - @TestOfyAndSql - void testSuccess_setsBasicSchedule() throws Exception { - assertThat(DatabaseMigrationStateSchedule.get()).isEqualTo(DEFAULT_TRANSITION_MAP); - assertThat(jpaTm().transact(() -> jpaTm().loadSingleton(DatabaseMigrationStateSchedule.class))) - .isEmpty(); - runCommandForced("--migration_schedule=1970-01-01T00:00:00.000Z=DATASTORE_ONLY"); - // use a raw ofy call to check what's in the DB - jpaTm() - .transact( - () -> - assertThat( - jpaTm() - .loadSingleton(DatabaseMigrationStateSchedule.class) - .get() - .migrationTransitions) - .isEqualTo(DEFAULT_TRANSITION_MAP)); - assertThat(DatabaseMigrationStateSchedule.get()).isEqualTo(DEFAULT_TRANSITION_MAP); - } - - @TestOfyAndSql - void testSuccess_fullSchedule() throws Exception { - DateTime now = fakeClock.nowUtc(); - DateTime datastorePrimary = now.plusHours(1); - DateTime datastorePrimaryNoAsync = now.plusHours(2); - DateTime datastorePrimaryReadOnly = now.plusHours(3); - DateTime sqlPrimary = now.plusHours(4); - DateTime sqlOnly = now.plusHours(5); - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY," - + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY," - + "%s=SQL_PRIMARY,%s=SQL_ONLY", - START_OF_TIME, - datastorePrimary, - datastorePrimaryNoAsync, - datastorePrimaryReadOnly, - sqlPrimary, - sqlOnly)); - assertThat(DatabaseMigrationStateSchedule.get().toValueMap()) - .containsExactlyEntriesIn( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - datastorePrimary, - MigrationState.DATASTORE_PRIMARY, - datastorePrimaryNoAsync, - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - datastorePrimaryReadOnly, - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - sqlPrimary, - MigrationState.SQL_PRIMARY, - sqlOnly, - MigrationState.SQL_ONLY)); - } - - @TestOfyAndSql - void testSuccess_warnsOnChangeSoon() throws Exception { - DateTime now = fakeClock.nowUtc(); - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY", - START_OF_TIME, now.plusMinutes(1))); - assertThat(DatabaseMigrationStateSchedule.get().toValueMap()) - .containsExactlyEntriesIn( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusMinutes(1), - MigrationState.DATASTORE_PRIMARY)); - assertInStdout("MAY BE DANGEROUS"); - } - - @TestOfyAndSql - void testSuccess_goesBackward() throws Exception { - DateTime now = fakeClock.nowUtc(); - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY," - + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY," - + "%s=DATASTORE_PRIMARY", - START_OF_TIME, now.plusHours(1), now.plusHours(2), now.plusHours(3), now.plusHours(4))); - assertThat(DatabaseMigrationStateSchedule.get().toValueMap()) - .containsExactlyEntriesIn( - ImmutableSortedMap.of( - START_OF_TIME, - MigrationState.DATASTORE_ONLY, - now.plusHours(1), - MigrationState.DATASTORE_PRIMARY, - now.plusHours(2), - MigrationState.DATASTORE_PRIMARY_NO_ASYNC, - now.plusHours(3), - MigrationState.DATASTORE_PRIMARY_READ_ONLY, - now.plusHours(4), - MigrationState.DATASTORE_PRIMARY)); - } - - @TestOfyAndSql - void testFailure_invalidTransition() { - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY_READ_ONLY", - START_OF_TIME, START_OF_TIME.plusHours(1)))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "validStateTransitions map cannot transition from DATASTORE_ONLY " - + "to DATASTORE_PRIMARY_READ_ONLY."); - } - - @TestOfyAndSql - void testFailure_invalidTransitionFromOldToNew() { - // The map we pass in is valid by itself, but we can't go from DATASTORE_ONLY now to - // DATASTORE_PRIMARY_READ_ONLY now - DateTime now = fakeClock.nowUtc(); - IllegalArgumentException thrown = - assertThrows( - IllegalArgumentException.class, - () -> - runCommandForced( - String.format( - "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY," - + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY", - START_OF_TIME, now.minusHours(3), now.minusHours(2), now.minusHours(1)))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "Cannot transition from current state-as-of-now DATASTORE_ONLY " - + "to new state-as-of-now DATASTORE_PRIMARY_READ_ONLY"); - } - - @TestOfyAndSql - void testFailure_invalidParams() { - assertThrows(ParameterException.class, this::runCommandForced); - assertThrows(ParameterException.class, () -> runCommandForced("--migration_schedule=FOOBAR")); - assertThrows( - ParameterException.class, - () -> runCommandForced("--migration_schedule=1970-01-01T00:00:00.000Z=FOOBAR")); - } -} diff --git a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java index 32e709bfb..8c43d8045 100644 --- a/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/GenerateZoneFilesActionTest.java @@ -38,17 +38,24 @@ import google.registry.model.eppcommon.StatusValue; import google.registry.model.host.HostResource; import google.registry.persistence.VKey; import google.registry.testing.FakeClock; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import java.net.InetAddress; import java.util.Map; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Duration; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link GenerateZoneFilesAction}. */ class GenerateZoneFilesActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); @Test diff --git a/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java b/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java index 082d968a2..f15fadc27 100644 --- a/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java +++ b/core/src/test/java/google/registry/tools/server/KillAllCommitLogsActionTest.java @@ -39,13 +39,20 @@ import google.registry.model.ofy.CommitLogCheckpointRoot; import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogMutation; import google.registry.testing.FakeResponse; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import org.joda.time.DateTime; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link KillAllCommitLogsAction}. */ class KillAllCommitLogsActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final ImmutableList> AFFECTED_TYPES = ImmutableList.of( CommitLogBucket.class, diff --git a/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java b/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java index 31193c6c4..4827d9f22 100644 --- a/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/KillAllEppResourcesActionTest.java @@ -52,15 +52,22 @@ import google.registry.model.poll.PollMessage; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.DatabaseHelper; import google.registry.testing.FakeResponse; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import java.util.stream.Stream; import org.joda.money.CurrencyUnit; import org.joda.money.Money; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Tests for {@link KillAllEppResourcesAction}. */ class KillAllEppResourcesActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final ImmutableSet AFFECTED_KINDS = Stream.of( EppResourceIndex.class, diff --git a/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java b/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java index 1c310459f..92b35995d 100644 --- a/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java +++ b/core/src/test/java/google/registry/tools/server/ResaveAllHistoryEntriesActionTest.java @@ -29,13 +29,20 @@ import google.registry.model.domain.DomainBase; import google.registry.model.domain.DomainHistory; import google.registry.model.reporting.HistoryEntry; import google.registry.testing.FakeResponse; +import google.registry.testing.TmOverrideExtension; import google.registry.testing.mapreduce.MapreduceTestCase; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; /** Unit tests for {@link ResaveAllHistoryEntriesAction}. */ class ResaveAllHistoryEntriesActionTest extends MapreduceTestCase { + @RegisterExtension + @Order(Order.DEFAULT - 1) + TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withOfy(); + private static final DatastoreService datastoreService = DatastoreServiceFactory.getDatastoreService(); diff --git a/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java b/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java index 322f9dc7d..b6bfcea22 100644 --- a/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java +++ b/core/src/test/java/google/registry/ui/server/registrar/SecuritySettingsTest.java @@ -16,6 +16,7 @@ package google.registry.ui.server.registrar; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth8.assertThat; +import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects; import static google.registry.testing.CertificateSamples.SAMPLE_CERT; import static google.registry.testing.CertificateSamples.SAMPLE_CERT2; import static google.registry.testing.CertificateSamples.SAMPLE_CERT2_HASH; @@ -48,13 +49,19 @@ class SecuritySettingsTest extends RegistrarSettingsActionTestCase { .asBuilder() .setClientCertificate(SAMPLE_CERT3, clock.nowUtc()) .build(); - Map response = action.handleJsonRequest(ImmutableMap.of( - "op", "update", - "id", CLIENT_ID, - "args", modified.toJsonMap())); + Map modifiedJsonMap = modified.toJsonMap(); + Map response = + action.handleJsonRequest( + ImmutableMap.of( + "op", "update", + "id", CLIENT_ID, + "args", modifiedJsonMap)); + modifiedJsonMap.put("lastUpdateTime", clock.nowUtc().toString()); assertThat(response).containsEntry("status", "SUCCESS"); - assertThat(response).containsEntry("results", ImmutableList.of(modified.toJsonMap())); - assertThat(loadRegistrar(CLIENT_ID)).isEqualTo(modified); + assertThat(response).containsEntry("results", ImmutableList.of(modifiedJsonMap)); + assertAboutImmutableObjects() + .that(loadRegistrar(CLIENT_ID)) + .isEqualExceptFields(modified, "lastUpdateTime"); assertMetric(CLIENT_ID, "update", "[OWNER]", "SUCCESS"); verifyNotificationEmailsSent(); } diff --git a/core/src/test/java/google/registry/webdriver/TestServerExtension.java b/core/src/test/java/google/registry/webdriver/TestServerExtension.java index 80b51d0df..67f570e10 100644 --- a/core/src/test/java/google/registry/webdriver/TestServerExtension.java +++ b/core/src/test/java/google/registry/webdriver/TestServerExtension.java @@ -30,8 +30,6 @@ import google.registry.server.Fixture; import google.registry.server.Route; import google.registry.server.TestServer; import google.registry.testing.AppEngineExtension; -import google.registry.testing.DatabaseHelper; -import google.registry.testing.FakeClock; import google.registry.testing.UserInfo; import java.net.URL; import java.net.UnknownHostException; @@ -43,8 +41,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingDeque; import javax.servlet.Filter; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -211,8 +207,6 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC // Clear the SQL database and set it as primary (we have to do this out of band because the // AppEngineExtension can't natively do it for us yet due to remaining ofy dependencies) new JpaTestExtensions.Builder().buildIntegrationTestExtension().beforeEach(null); - DatabaseHelper.setMigrationScheduleToSqlPrimary( - new FakeClock(DateTime.now(DateTimeZone.UTC))); // sleep a few millis to make sure we get to SQL-primary mode Thread.sleep(4); AppEngineExtension.loadInitialData(); diff --git a/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java b/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java index 28339f496..a017ea707 100644 --- a/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java +++ b/core/src/test/java/google/registry/whois/RegistrarWhoisResponseTest.java @@ -110,8 +110,8 @@ class RegistrarWhoisResponseTest { .setTypes(ImmutableSet.of(RegistrarContact.Type.TECH)) .setVisibleInWhoisAsTech(true) .build()); - persistSimpleResources(contacts); persistResource(registrar); + persistSimpleResources(contacts); RegistrarWhoisResponse registrarWhoisResponse = new RegistrarWhoisResponse(registrar, clock.nowUtc()); diff --git a/core/src/test/resources/google/registry/module/backend/backend_routing.txt b/core/src/test/resources/google/registry/module/backend/backend_routing.txt index 0aaa6f78b..7f2ebf4ed 100644 --- a/core/src/test/resources/google/registry/module/backend/backend_routing.txt +++ b/core/src/test/resources/google/registry/module/backend/backend_routing.txt @@ -30,7 +30,6 @@ PATH CLASS /_dr/task/rdeReport RdeReportAction POST n INTERNAL,API APP ADMIN /_dr/task/rdeStaging RdeStagingAction GET,POST n INTERNAL,API APP ADMIN /_dr/task/rdeUpload RdeUploadAction POST n INTERNAL,API APP ADMIN -/_dr/task/refreshDnsOnHostRename RefreshDnsOnHostRenameAction GET n INTERNAL,API APP ADMIN /_dr/task/relockDomain RelockDomainAction POST y INTERNAL,API APP ADMIN /_dr/task/resaveAllEppResources ResaveAllEppResourcesAction GET n INTERNAL,API APP ADMIN /_dr/task/resaveAllEppResourcesPipeline ResaveAllEppResourcesPipelineAction GET n INTERNAL,API APP ADMIN diff --git a/docs/flows.md b/docs/flows.md index d05f2fd46..5a9f2cb17 100644 --- a/docs/flows.md +++ b/docs/flows.md @@ -31,8 +31,6 @@ An EPP flow that creates a new contact. * Resource with this id already exists. * 2306 * Declining contact disclosure is disallowed by server policy. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactDeleteFlow @@ -59,8 +57,6 @@ or failure message when the process is complete. * Resource status prohibits this operation. * 2305 * Resource to be deleted has active incoming references. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactInfoFlow @@ -107,8 +103,6 @@ transfer request, which then becomes effective immediately. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactTransferCancelFlow @@ -134,8 +128,6 @@ request. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactTransferQueryFlow @@ -186,8 +178,6 @@ that window, this flow allows the losing client to reject the transfer request. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactTransferRequestFlow @@ -216,8 +206,6 @@ or rejected, and the gaining registrar can also cancel the transfer request. * Resource with this id does not exist. * 2304 * Resource status prohibits this operation. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## ContactUpdateFlow @@ -244,8 +232,6 @@ An EPP flow that updates a contact. * 2306 * Cannot add and remove the same value. * Declining contact disclosure is disallowed by server policy. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainCheckFlow @@ -410,8 +396,6 @@ An EPP flow that creates a new domain resource. * Too many nameservers set on this domain. * Domain labels cannot end with a dash. * Only encoded signed marks are supported. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainDeleteFlow @@ -436,8 +420,6 @@ An EPP flow that deletes a domain. * Resource status prohibits this operation. * 2305 * Domain to be deleted has subordinate hosts. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainInfoFlow @@ -514,8 +496,6 @@ comes in at the exact millisecond that the domain would have expired. * 2306 * Periods for domain registrations must be specified in years. * The requested fees cannot be provided in the requested currency. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainRestoreRequestFlow @@ -577,8 +557,6 @@ regardless of what the original expiration time was. * Domain is not eligible for restore. * 2306 * The requested fees cannot be provided in the requested currency. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferApproveFlow @@ -610,8 +588,6 @@ replaced with new ones with the correct approval time. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferCancelFlow @@ -642,8 +618,6 @@ transfer period passed. In this flow, those speculative objects are deleted. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferQueryFlow @@ -699,8 +673,6 @@ transfer period passed. In this flow, those speculative objects are deleted. * The resource does not have a pending transfer. * 2303 * Resource with this id does not exist. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainTransferRequestFlow @@ -762,8 +734,6 @@ new ones with the correct approval time). EPP extension. * Periods for domain registrations must be specified in years. * The requested fees cannot be provided in the requested currency. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## DomainUpdateFlow @@ -822,8 +792,6 @@ statuses are updated at once. * The secDNS:all element must have value 'true' if present. * Too many DS records set on a domain. * Too many nameservers set on this domain. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## HelloFlow @@ -886,8 +854,6 @@ allows creating a host name, and if necessary enqueues tasks to update DNS. * Superordinate domain for this hostname is in pending delete. * 2306 * Host names must be at least two levels below the registry suffix. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## HostDeleteFlow @@ -918,8 +884,6 @@ or failure message when the process is complete. * Resource status prohibits this operation. * 2305 * Resource to be deleted has active incoming references. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## HostInfoFlow @@ -992,8 +956,6 @@ are enqueued to update DNS accordingly. * 2306 * Cannot add and remove the same value. * Host names must be at least two levels below the registry suffix. -* 2400 - * Registry is currently undergoing maintenance and is in read-only mode. ## LoginFlow diff --git a/release/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml index df2854309..61b0c6b74 100644 --- a/release/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -88,8 +88,6 @@ steps: beam_pipeline_common \ ${TAG_NAME} \ ${PROJECT_ID} \ - google.registry.beam.initsql.InitSqlPipeline \ - google/registry/beam/init_sql_pipeline_metadata.json \ google.registry.beam.datastore.BulkDeleteDatastorePipeline \ google/registry/beam/bulk_delete_datastore_pipeline_metadata.json \ google.registry.beam.spec11.Spec11Pipeline \