diff --git a/core/build.gradle b/core/build.gradle index 1e9c50f18..d13fd4b44 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -693,10 +693,6 @@ createToolTask( 'google.registry.tools.DevTool', sourceSets.nonprod) -createToolTask( - 'createSyntheticDomainHistories', - 'google.registry.tools.javascrap.CreateSyntheticDomainHistoriesPipeline') - project.tasks.create('generateSqlSchema', JavaExec) { classpath = sourceSets.nonprod.runtimeClasspath main = 'google.registry.tools.DevTool' @@ -766,6 +762,11 @@ if (environment == 'alpha') { mainClass: 'google.registry.beam.resave.ResaveAllEppResourcesPipeline', metaData: 'google/registry/beam/resave_all_epp_resources_pipeline_metadata.json' ], + wipeOutContactHistoryPii: + [ + mainClass: 'google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline', + metaData: 'google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json' + ], ] project.tasks.create("stageBeamPipelines") { doLast { diff --git a/core/src/main/java/google/registry/batch/BatchModule.java b/core/src/main/java/google/registry/batch/BatchModule.java index 1272684c5..b7471b178 100644 --- a/core/src/main/java/google/registry/batch/BatchModule.java +++ b/core/src/main/java/google/registry/batch/BatchModule.java @@ -117,6 +117,12 @@ public class BatchModule { return extractOptionalDatetimeParameter(req, ExpandRecurringBillingEventsAction.PARAM_END_TIME); } + @Provides + @Parameter(WipeOutContactHistoryPiiAction.PARAM_CUTOFF_TIME) + static Optional provideCutoffTime(HttpServletRequest req) { + return extractOptionalDatetimeParameter(req, WipeOutContactHistoryPiiAction.PARAM_CUTOFF_TIME); + } + @Provides @Parameter(ExpandRecurringBillingEventsAction.PARAM_ADVANCE_CURSOR) static boolean provideAdvanceCursor(HttpServletRequest req) { diff --git a/core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java b/core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java index c4851b4d8..5ff58cbb2 100644 --- a/core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java +++ b/core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java @@ -14,31 +14,39 @@ package google.registry.batch; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; -import static org.apache.http.HttpStatus.SC_OK; +import static google.registry.batch.BatchModule.PARAM_DRY_RUN; +import static google.registry.beam.BeamUtils.createJobName; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.common.annotations.VisibleForTesting; +import com.google.api.services.dataflow.Dataflow; +import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter; +import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest; +import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse; +import com.google.common.collect.ImmutableMap; import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; +import google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline; import google.registry.config.RegistryConfig.Config; +import google.registry.config.RegistryEnvironment; import google.registry.model.contact.ContactHistory; import google.registry.request.Action; import google.registry.request.Action.Service; +import google.registry.request.Parameter; import google.registry.request.Response; import google.registry.request.auth.Auth; import google.registry.util.Clock; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; +import java.io.IOException; +import java.util.Optional; import javax.inject.Inject; import org.joda.time.DateTime; /** - * An action that wipes out Personal Identifiable Information (PII) fields of {@link ContactHistory} - * entities. + * An action that launches {@link WipeOutContactHistoryPiiPipeline} to wipe out Personal + * Identifiable Information (PII) fields of {@link ContactHistory} entities. * - *

ContactHistory entities should be retained in the database for only certain amount of time. - * This periodic wipe out action only applies to SQL. + *

{@link ContactHistory} entities should be retained in the database for only certain amount of + * time. */ @Action( service = Service.BACKEND, @@ -47,90 +55,89 @@ import org.joda.time.DateTime; public class WipeOutContactHistoryPiiAction implements Runnable { public static final String PATH = "/_dr/task/wipeOutContactHistoryPii"; + public static final String PARAM_CUTOFF_TIME = "wipeoutTime"; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final String PIPELINE_NAME = "wipe_out_contact_history_pii_pipeline"; + private final Clock clock; - private final Response response; + private final boolean isDryRun; + private final Optional maybeCutoffTime; private final int minMonthsBeforeWipeOut; - private final int wipeOutQueryBatchSize; + private final String stagingBucketUrl; + private final String projectId; + private final String jobRegion; + private final Dataflow dataflow; + private final Response response; @Inject public WipeOutContactHistoryPiiAction( Clock clock, + @Parameter(PARAM_DRY_RUN) boolean isDryRun, + @Parameter(PARAM_CUTOFF_TIME) Optional maybeCutoffTime, @Config("minMonthsBeforeWipeOut") int minMonthsBeforeWipeOut, - @Config("wipeOutQueryBatchSize") int wipeOutQueryBatchSize, + @Config("beamStagingBucketUrl") String stagingBucketUrl, + @Config("projectId") String projectId, + @Config("defaultJobRegion") String jobRegion, + Dataflow dataflow, Response response) { this.clock = clock; - this.response = response; + this.isDryRun = isDryRun; + this.maybeCutoffTime = maybeCutoffTime; this.minMonthsBeforeWipeOut = minMonthsBeforeWipeOut; - this.wipeOutQueryBatchSize = wipeOutQueryBatchSize; + this.stagingBucketUrl = stagingBucketUrl; + this.projectId = projectId; + this.jobRegion = jobRegion; + this.dataflow = dataflow; + this.response = response; } @Override public void run() { response.setContentType(MediaType.PLAIN_TEXT_UTF_8); + DateTime cutoffTime = + maybeCutoffTime.orElse(clock.nowUtc().minusMonths(minMonthsBeforeWipeOut)); + LaunchFlexTemplateParameter launchParameter = + new LaunchFlexTemplateParameter() + .setJobName( + createJobName( + String.format( + "contact-history-pii-wipeout-%s", + cutoffTime.toString("yyyy-MM-dd't'HH-mm-ss'z'")), + clock)) + .setContainerSpecGcsPath( + String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME)) + .setParameters( + ImmutableMap.of( + "registryEnvironment", + RegistryEnvironment.get().name(), + "cutoffTime", + cutoffTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"), + "isDryRun", + Boolean.toString(isDryRun))); + logger.atInfo().log( + "Launching Beam pipeline to wipe out all PII of contact history entities prior to %s%s.", + cutoffTime, " in dry run mode"); try { - int totalNumOfWipedEntities = 0; - DateTime wipeOutTime = clock.nowUtc().minusMonths(minMonthsBeforeWipeOut); - logger.atInfo().log( - "About to wipe out all PII of contact history entities prior to %s.", wipeOutTime); - - int numOfWipedEntities = 0; - do { - numOfWipedEntities = - tm().transact( - () -> - wipeOutContactHistoryData( - getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime))); - totalNumOfWipedEntities += numOfWipedEntities; - } while (numOfWipedEntities > 0); - String msg = - String.format( - "Done. Wiped out PII of %d ContactHistory entities in total.", - totalNumOfWipedEntities); - logger.atInfo().log(msg); - response.setPayload(msg); + LaunchFlexTemplateResponse launchResponse = + dataflow + .projects() + .locations() + .flexTemplates() + .launch( + projectId, + jobRegion, + new LaunchFlexTemplateRequest().setLaunchParameter(launchParameter)) + .execute(); + logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString()); response.setStatus(SC_OK); - - } catch (Exception e) { - logger.atSevere().withCause(e).log( - "Exception thrown during the process of wiping out contact history PII."); - response.setStatus(SC_INTERNAL_SERVER_ERROR); response.setPayload( String.format( - "Exception thrown during the process of wiping out contact history PII with cause" - + ": %s", - e)); + "Launched contact history PII wipeout pipeline: %s", + launchResponse.getJob().getId())); + } catch (IOException e) { + logger.atWarning().withCause(e).log("Pipeline Launch failed"); + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage())); } } - - /** - * Returns a stream of up to {@link #wipeOutQueryBatchSize} {@link ContactHistory} entities - * containing PII that are prior to @param wipeOutTime. - */ - @VisibleForTesting - Stream getNextContactHistoryEntitiesWithPiiBatch(DateTime wipeOutTime) { - // email is one of the required fields in EPP, meaning it's initially not null. - // Therefore, checking if it's null is one way to avoid processing contact history entities - // that have been processed previously. Refer to RFC 5733 for more information. - return tm().query( - "FROM ContactHistory WHERE modificationTime < :wipeOutTime " + "AND email IS NOT NULL", - ContactHistory.class) - .setParameter("wipeOutTime", wipeOutTime) - .setMaxResults(wipeOutQueryBatchSize) - .getResultStream(); - } - - /** Wipes out the PII of each of the {@link ContactHistory} entities in the stream. */ - @VisibleForTesting - int wipeOutContactHistoryData(Stream contactHistoryEntities) { - AtomicInteger numOfEntities = new AtomicInteger(0); - contactHistoryEntities.forEach( - contactHistoryEntity -> { - tm().update(contactHistoryEntity.asBuilder().wipeOutPii().build()); - numOfEntities.incrementAndGet(); - }); - logger.atInfo().log( - "Wiped out all PII fields of %d ContactHistory entities.", numOfEntities.get()); - return numOfEntities.get(); - } } diff --git a/core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipeline.java b/core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipeline.java new file mode 100644 index 000000000..270f92861 --- /dev/null +++ b/core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipeline.java @@ -0,0 +1,166 @@ +// Copyright 2023 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.beam.wipeout; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.persistence.transaction.TransactionManagerFactory.tm; +import static org.apache.beam.sdk.values.TypeDescriptors.voids; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Streams; +import google.registry.beam.common.RegistryJpaIO; +import google.registry.model.contact.ContactHistory; +import google.registry.model.reporting.HistoryEntry.HistoryEntryId; +import google.registry.persistence.PersistenceModule.TransactionIsolationLevel; +import google.registry.persistence.VKey; +import java.io.Serializable; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.coders.VarLongCoder; +import org.apache.beam.sdk.metrics.Counter; +import org.apache.beam.sdk.metrics.Metrics; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.transforms.join.CoGroupByKey; +import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TupleTag; +import org.joda.time.DateTime; + +/** + * Definition of a Dataflow Flex pipeline template, which finds out {@link ContactHistory} entries + * that are older than a given age (excluding the most recent one, even if it falls with the range) + * and wipe out PII information in them. + * + *

To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha \ + * --pipeline=wipeOutContactHistoryPii}. + * + *

Then, you can run the staged template via the API client library, gCloud or a raw REST call. + */ +public class WipeOutContactHistoryPiiPipeline implements Serializable { + + private static final long serialVersionUID = -4111052675715913820L; + private static final TupleTag REVISIONS_TO_WIPE = new TupleTag<>(); + private static final TupleTag MOST_RECENT_REVISION = new TupleTag<>(); + + private final DateTime cutoffTime; + private final boolean dryRun; + private final Counter contactsInScope = + Metrics.counter("WipeOutContactHistoryPii", "contacts in scope"); + private final Counter historiesToWipe = + Metrics.counter("WipeOutContactHistoryPii", "contact histories to wipe PII from"); + private final Counter historiesWiped = + Metrics.counter("WipeOutContactHistoryPii", "contact histories actually updated"); + + WipeOutContactHistoryPiiPipeline(WipeOutContactHistoryPiiPipelineOptions options) { + dryRun = options.getIsDryRun(); + cutoffTime = DateTime.parse(options.getCutoffTime()); + } + + void setup(Pipeline pipeline) { + KeyedPCollectionTuple.of(REVISIONS_TO_WIPE, getHistoryEntriesToWipe(pipeline)) + .and(MOST_RECENT_REVISION, getMostRecentHistoryEntries(pipeline)) + .apply("Group by contact", CoGroupByKey.create()) + .apply( + "Wipe out PII", + MapElements.into(voids()) + .via( + kv -> { + String repoId = kv.getKey(); + long mostRecentRevision = kv.getValue().getOnly(MOST_RECENT_REVISION); + ImmutableList revisionsToWipe = + Streams.stream(kv.getValue().getAll(REVISIONS_TO_WIPE)) + .filter(e -> e != mostRecentRevision) + .collect(toImmutableList()); + if (revisionsToWipe.isEmpty()) { + return null; + } + contactsInScope.inc(); + tm().transact( + () -> { + for (long revisionId : revisionsToWipe) { + historiesToWipe.inc(); + ContactHistory history = + tm().loadByKey( + VKey.create( + ContactHistory.class, + new HistoryEntryId(repoId, revisionId))); + // In the unlikely case where multiple pipelines run at the + // same time, or where the runner decides to rerun a particular + // transform, we might have a history entry that has already been + // wiped at this point. There's no need to wipe it again. + if (!dryRun + && history.getContactBase().isPresent() + && history.getContactBase().get().getEmailAddress() != null) { + historiesWiped.inc(); + tm().update(history.asBuilder().wipeOutPii().build()); + } + } + }); + return null; + })); + } + + PCollection> getHistoryEntriesToWipe(Pipeline pipeline) { + return pipeline.apply( + "Find contact histories to wipee", + // Email is one of the required fields in EPP, meaning it's initially not null when it + // is set by EPP flows (even though it is nullalbe in the SQL schema). Therefore, + // checking if it's null is one way to avoid processing contact history entities that + // have been processed previously. Refer to RFC 5733 for more information. + RegistryJpaIO.read( + "SELECT repoId, revisionId FROM ContactHistory WHERE email IS NOT NULL AND" + + " modificationTime < :cutoffTime", + ImmutableMap.of("cutoffTime", cutoffTime), + Object[].class, + row -> KV.of((String) row[0], (long) row[1])) + .withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))); + } + + PCollection> getMostRecentHistoryEntries(Pipeline pipeline) { + return pipeline.apply( + "Find the most recent historiy entry for each contact", + RegistryJpaIO.read( + "SELECT repoId, revisionId FROM ContactHistory" + + " WHERE (repoId, modificationTime) IN" + + " (SELECT repoId, MAX(modificationTime) FROM ContactHistory GROUP BY repoId)", + ImmutableMap.of(), + Object[].class, + row -> KV.of((String) row[0], (long) row[1])) + .withCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))); + } + + PipelineResult run(Pipeline pipeline) { + setup(pipeline); + return pipeline.run(); + } + + public static void main(String[] args) { + PipelineOptionsFactory.register(WipeOutContactHistoryPiiPipelineOptions.class); + WipeOutContactHistoryPiiPipelineOptions options = + PipelineOptionsFactory.fromArgs(args) + .withValidation() + .as(WipeOutContactHistoryPiiPipelineOptions.class); + // Repeatable read should be more than enough since we are dealing with old history entries that + // are otherwise immutable. + options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ); + Pipeline pipeline = Pipeline.create(options); + new WipeOutContactHistoryPiiPipeline(options).run(pipeline); + } +} diff --git a/core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipelineOptions.java b/core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipelineOptions.java new file mode 100644 index 000000000..833b267bf --- /dev/null +++ b/core/src/main/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipelineOptions.java @@ -0,0 +1,37 @@ +// Copyright 2023 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.beam.wipeout; + +import google.registry.beam.common.RegistryPipelineOptions; +import org.apache.beam.sdk.options.Default; +import org.apache.beam.sdk.options.Description; + +public interface WipeOutContactHistoryPiiPipelineOptions extends RegistryPipelineOptions { + + @Description( + "A contact history entry with a history modification time before this time will have its PII" + + " wiped, unless it is the most entry for the contact.") + String getCutoffTime(); + + void setCutoffTime(String value); + + @Description( + "If true, the wiped out billing events will not be saved but the pipeline metrics counter" + + " will still be updated.") + @Default.Boolean(false) + boolean getIsDryRun(); + + void setIsDryRun(boolean value); +} diff --git a/core/src/main/java/google/registry/config/RegistryConfig.java b/core/src/main/java/google/registry/config/RegistryConfig.java index 8250cf532..60c151ca5 100644 --- a/core/src/main/java/google/registry/config/RegistryConfig.java +++ b/core/src/main/java/google/registry/config/RegistryConfig.java @@ -1325,12 +1325,6 @@ public final class RegistryConfig { return config.contactHistory.minMonthsBeforeWipeOut; } - @Provides - @Config("wipeOutQueryBatchSize") - public static int provideWipeOutQueryBatchSize(RegistryConfigSettings config) { - return config.contactHistory.wipeOutQueryBatchSize; - } - @Provides @Config("jdbcBatchSize") public static int provideHibernateJdbcBatchSize(RegistryConfigSettings config) { diff --git a/core/src/main/java/google/registry/config/RegistryConfigSettings.java b/core/src/main/java/google/registry/config/RegistryConfigSettings.java index f3701d660..741e7897f 100644 --- a/core/src/main/java/google/registry/config/RegistryConfigSettings.java +++ b/core/src/main/java/google/registry/config/RegistryConfigSettings.java @@ -243,7 +243,6 @@ public class RegistryConfigSettings { /** Configuration for contact history. */ public static class ContactHistory { public int minMonthsBeforeWipeOut; - public int wipeOutQueryBatchSize; } /** Configuration for dns update. */ diff --git a/core/src/main/java/google/registry/config/files/default-config.yaml b/core/src/main/java/google/registry/config/files/default-config.yaml index dedb9826c..6f7f19924 100644 --- a/core/src/main/java/google/registry/config/files/default-config.yaml +++ b/core/src/main/java/google/registry/config/files/default-config.yaml @@ -474,8 +474,6 @@ registryTool: contactHistory: # The number of months that a ContactHistory entity should be stored in the database. minMonthsBeforeWipeOut: 18 - # The batch size for querying ContactHistory table in the database. - wipeOutQueryBatchSize: 500 # Configuration options relevant to the DNS update functionality. dnsUpdate: diff --git a/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml b/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml index c63e53992..404cb7222 100644 --- a/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml +++ b/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml @@ -280,6 +280,6 @@ This job runs weekly to wipe out PII fields of ContactHistory entities that have been in the database for a certain period of time. - 0 15 * 12 1 + 0 15 * * 1 diff --git a/core/src/main/resources/google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json b/core/src/main/resources/google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json new file mode 100644 index 000000000..7fe61f46f --- /dev/null +++ b/core/src/main/resources/google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json @@ -0,0 +1,39 @@ +{ + "name": "Wipe Out PII From Old Contact History Entries", + "description": "An Apache Beam batch pipeline that finds old contact history entries and remove PII information from them.", + "parameters": [ + { + "name": "registryEnvironment", + "label": "The Registry environment.", + "helpText": "The Registry environment.", + "is_optional": false, + "regexes": [ + "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$" + ] + }, + { + "name": "isolationOverride", + "label": "The desired SQL transaction isolation level.", + "helpText": "The desired SQL transaction isolation level.", + "is_optional": true, + "regexes": [ + "^[0-9A-Z_]+$" + ] + }, + { + "name": "cutoffTime", + "label": "The maximum history modification time of a contact history entry eligible for wipe out.", + "helpText": "If the history modificaiton time of contact history entry is older than this, and it is not the most recent entry of a contact, it will have its PII wiped out.", + "is_optional": true + }, + { + "name": "isDryRun", + "label": "Whether this job is a dry run.", + "helpText": "If true, no changes will be saved to the database.", + "is_optional": true, + "regexes": [ + "^true|false$" + ] + } + ] +} diff --git a/core/src/test/java/google/registry/batch/WipeOutContactHistoryPiiActionTest.java b/core/src/test/java/google/registry/batch/WipeOutContactHistoryPiiActionTest.java index 008a520f5..0efe68c07 100644 --- a/core/src/test/java/google/registry/batch/WipeOutContactHistoryPiiActionTest.java +++ b/core/src/test/java/google/registry/batch/WipeOutContactHistoryPiiActionTest.java @@ -15,348 +15,119 @@ package google.registry.batch; import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; -import static com.google.common.truth.Truth8.assertThat; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; -import static google.registry.testing.DatabaseHelper.persistResource; -import static org.apache.http.HttpStatus.SC_OK; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import google.registry.model.contact.Contact; -import google.registry.model.contact.ContactAddress; -import google.registry.model.contact.ContactAuthInfo; -import google.registry.model.contact.ContactBase; -import google.registry.model.contact.ContactHistory; -import google.registry.model.contact.ContactPhoneNumber; -import google.registry.model.contact.Disclose; -import google.registry.model.contact.PostalInfo; -import google.registry.model.eppcommon.AuthInfo.PasswordAuth; -import google.registry.model.eppcommon.PresenceMarker; -import google.registry.model.eppcommon.StatusValue; -import google.registry.persistence.transaction.JpaTestExtensions; -import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; -import google.registry.testing.DatabaseHelper; +import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest; +import google.registry.beam.BeamActionTestBase; import google.registry.testing.FakeClock; -import google.registry.testing.FakeResponse; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; /** Unit tests for {@link WipeOutContactHistoryPiiAction}. */ -class WipeOutContactHistoryPiiActionTest { +class WipeOutContactHistoryPiiActionTest extends BeamActionTestBase { - private static final int TEST_BATCH_SIZE = 20; - private static final int MIN_MONTHS_BEFORE_WIPE_OUT = 18; - private static final Contact DEFAULT_CONTACT = - new Contact.Builder() - .setContactId("sh8013") - .setRepoId("2FF-ROID") - .setStatusValues(ImmutableSet.of(StatusValue.CLIENT_DELETE_PROHIBITED)) - .setLocalizedPostalInfo( - new PostalInfo.Builder() - .setType(PostalInfo.Type.LOCALIZED) - .setAddress( - new ContactAddress.Builder() - .setStreet(ImmutableList.of("123 Grand Ave")) - .build()) - .build()) - .setInternationalizedPostalInfo( - new PostalInfo.Builder() - .setType(PostalInfo.Type.INTERNATIONALIZED) - .setName("John Doe") - .setOrg("Example Inc.") - .setAddress( - new ContactAddress.Builder() - .setStreet(ImmutableList.of("123 Example Dr.", "Suite 100")) - .setCity("Dulles") - .setState("VA") - .setZip("20166-6503") - .setCountryCode("US") - .build()) - .build()) - .setVoiceNumber( - new ContactPhoneNumber.Builder() - .setPhoneNumber("+1.7035555555") - .setExtension("1234") - .build()) - .setFaxNumber(new ContactPhoneNumber.Builder().setPhoneNumber("+1.7035555556").build()) - .setEmailAddress("jdoe@example.com") - .setPersistedCurrentSponsorRegistrarId("TheRegistrar") - .setCreationRegistrarId("NewRegistrar") - .setLastEppUpdateRegistrarId("NewRegistrar") - .setCreationTimeForTest(DateTime.parse("1999-04-03T22:00:00.0Z")) - .setLastEppUpdateTime(DateTime.parse("1999-12-03T09:00:00.0Z")) - .setLastTransferTime(DateTime.parse("2000-04-08T09:00:00.0Z")) - .setAuthInfo(ContactAuthInfo.create(PasswordAuth.create("2fooBAR"))) - .setDisclose( - new Disclose.Builder() - .setFlag(true) - .setVoice(new PresenceMarker()) - .setEmail(new PresenceMarker()) - .build()) - .build(); - - @RegisterExtension - final JpaIntegrationTestExtension jpa = - new JpaTestExtensions.Builder().buildIntegrationTestExtension(); - - private final FakeClock clock = new FakeClock(DateTime.parse("2021-08-26T20:21:22Z")); - - private FakeResponse response; - private WipeOutContactHistoryPiiAction action; + private final DateTime now = DateTime.parse("2019-01-19T01:02:03Z"); + private final FakeClock clock = new FakeClock(now); + private final Map expectedParameters = new HashMap<>(); + private final ArgumentCaptor launchRequest = + ArgumentCaptor.forClass(LaunchFlexTemplateRequest.class); + private WipeOutContactHistoryPiiAction action = + new WipeOutContactHistoryPiiAction( + clock, + false, + Optional.empty(), + 8, + "tucketBucket", + "testProject", + "testRegion", + dataflow, + response); @BeforeEach - void beforeEach() { - response = new FakeResponse(); + void before() { + expectedParameters.put("registryEnvironment", "UNITTEST"); + expectedParameters.put("isDryRun", "false"); + expectedParameters.put("cutoffTime", "2018-05-19T01:02:03.000Z"); + } + + @Test + void testSuccess() throws Exception { + action.run(); + assertThat(response.getStatus()).isEqualTo(200); + assertThat(response.getPayload()) + .isEqualTo("Launched contact history PII wipeout pipeline: jobid"); + verify(templates, times(1)) + .launch(eq("testProject"), eq("testRegion"), launchRequest.capture()); + assertThat(launchRequest.getValue().getLaunchParameter().getParameters()) + .containsExactlyEntriesIn(expectedParameters); + } + + @Test + void testSuccess_providedCutoffTime() throws Exception { action = new WipeOutContactHistoryPiiAction( - clock, MIN_MONTHS_BEFORE_WIPE_OUT, TEST_BATCH_SIZE, response); - } - - @Test - void getAllHistoryEntitiesOlderThan_returnsAllPersistedEntities() { - ImmutableList expectedToBeWipedOut = - persistLotsOfContactHistoryEntities(20, MIN_MONTHS_BEFORE_WIPE_OUT + 1, 0, DEFAULT_CONTACT); - tm().transact( - () -> - assertThat( - action.getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT))) - .containsExactlyElementsIn(expectedToBeWipedOut)); - } - - @Test - void getAllHistoryEntitiesOlderThan_returnsOnlyOldEnoughPersistedEntities() { - ImmutableList expectedToBeWipedOut = - persistLotsOfContactHistoryEntities(19, MIN_MONTHS_BEFORE_WIPE_OUT + 2, 0, DEFAULT_CONTACT); - - // persisted entities that should not be part of the actual result - persistLotsOfContactHistoryEntities(15, 17, MIN_MONTHS_BEFORE_WIPE_OUT - 1, DEFAULT_CONTACT); - - tm().transact( - () -> - assertThat( - action.getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT))) - .containsExactlyElementsIn(expectedToBeWipedOut)); - } - - @Test - void run_withNoEntitiesToWipeOut_success() { - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(0); + clock, + false, + Optional.of(now.minusYears(1)), + 8, + "tucketBucket", + "testProject", + "testRegion", + dataflow, + response); action.run(); - - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(0); - - assertThat(response.getStatus()).isEqualTo(SC_OK); + expectedParameters.put("cutoffTime", "2018-01-19T01:02:03.000Z"); + assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getPayload()) - .isEqualTo("Done. Wiped out PII of 0 ContactHistory entities in total."); + .isEqualTo("Launched contact history PII wipeout pipeline: jobid"); + verify(templates, times(1)) + .launch(eq("testProject"), eq("testRegion"), launchRequest.capture()); + assertThat(launchRequest.getValue().getLaunchParameter().getParameters()) + .containsExactlyEntriesIn(expectedParameters); } @Test - void run_withOneBatchOfEntities_success() { - int numOfMonthsFromNow = MIN_MONTHS_BEFORE_WIPE_OUT + 2; - ImmutableList expectedToBeWipedOut = - persistLotsOfContactHistoryEntities(20, numOfMonthsFromNow, 0, DEFAULT_CONTACT); - - // The query should return a stream of all persisted entities. - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(expectedToBeWipedOut.size()); - - assertAllEntitiesContainPii(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); - + void testSuccess_dryRun() throws Exception { + action = + new WipeOutContactHistoryPiiAction( + clock, + true, + Optional.empty(), + 8, + "tucketBucket", + "testProject", + "testRegion", + dataflow, + response); action.run(); + expectedParameters.put("isDryRun", "true"); + assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getPayload()) - .isEqualTo("Done. Wiped out PII of 20 ContactHistory entities in total."); - - // The query should return an empty stream after the wipe out action. - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(0); - - assertAllPiiFieldsAreWipedOut(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); + .isEqualTo("Launched contact history PII wipeout pipeline: jobid"); + verify(templates, times(1)) + .launch(eq("testProject"), eq("testRegion"), launchRequest.capture()); + assertThat(launchRequest.getValue().getLaunchParameter().getParameters()) + .containsExactlyEntriesIn(expectedParameters); } @Test - void run_withMultipleBatches_numOfEntitiesAsNonMultipleOfBatchSize_success() { - int numOfMonthsFromNow = MIN_MONTHS_BEFORE_WIPE_OUT + 2; - ImmutableList expectedToBeWipedOut = - persistLotsOfContactHistoryEntities(56, numOfMonthsFromNow, 0, DEFAULT_CONTACT); - - // The query should return a subset of all persisted data. - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(TEST_BATCH_SIZE); - - assertAllEntitiesContainPii(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); + void testFailure_launchError() throws Exception { + when(launch.execute()).thenThrow(new IOException("cannot launch")); action.run(); - assertThat(response.getPayload()) - .isEqualTo("Done. Wiped out PII of 56 ContactHistory entities in total."); - - // The query should return an empty stream after the wipe out action. - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(0); - - assertAllPiiFieldsAreWipedOut(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); - } - - @Test - void run_withMultipleBatches_numOfEntitiesAsMultiplesOfBatchSize_success() { - int numOfMonthsFromNow = MIN_MONTHS_BEFORE_WIPE_OUT + 2; - ImmutableList expectedToBeWipedOut = - persistLotsOfContactHistoryEntities( - TEST_BATCH_SIZE * 2, numOfMonthsFromNow, 0, DEFAULT_CONTACT); - - // The query should return a subset of all persisted data. - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(TEST_BATCH_SIZE); - - assertAllEntitiesContainPii(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); - action.run(); - assertThat(response.getPayload()) - .isEqualTo("Done. Wiped out PII of 40 ContactHistory entities in total."); - - // The query should return an empty stream after the wipe out action. - assertThat( - tm().transact( - () -> - action - .getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)) - .count())) - .isEqualTo(0); - - assertAllPiiFieldsAreWipedOut(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); - } - - @Test - void wipeOutContactHistoryData_wipesOutNoEntity() { - tm().transact( - () -> - assertThat( - action.wipeOutContactHistoryData( - action.getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)))) - .isEqualTo(0)); - } - - @Test - void wipeOutContactHistoryData_wipesOutMultipleEntities() { - int numOfMonthsFromNow = MIN_MONTHS_BEFORE_WIPE_OUT + 3; - ImmutableList expectedToBeWipedOut = - persistLotsOfContactHistoryEntities(20, numOfMonthsFromNow, 0, DEFAULT_CONTACT); - - assertAllEntitiesContainPii(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); - - tm().transact( - () -> { - action.wipeOutContactHistoryData( - action.getNextContactHistoryEntitiesWithPiiBatch( - clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT))); - }); - - assertAllPiiFieldsAreWipedOut(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut)); - } - - /** persists a number of ContactHistory entities for load and query testing. */ - ImmutableList persistLotsOfContactHistoryEntities( - int numOfEntities, int minusMonths, int minusDays, Contact contact) { - ImmutableList.Builder expectedEntitiesBuilder = new ImmutableList.Builder<>(); - for (int i = 0; i < numOfEntities; i++) { - expectedEntitiesBuilder.add( - persistResource( - new ContactHistory() - .asBuilder() - .setRegistrarId("NewRegistrar") - .setModificationTime(clock.nowUtc().minusMonths(minusMonths).minusDays(minusDays)) - .setType(ContactHistory.Type.CONTACT_DELETE) - .setContact(persistResource(contact)) - .build())); - } - return expectedEntitiesBuilder.build(); - } - - boolean areAllPiiFieldsWiped(ContactBase contactBase) { - return contactBase.getEmailAddress() == null - && contactBase.getFaxNumber() == null - && contactBase.getInternationalizedPostalInfo() == null - && contactBase.getLocalizedPostalInfo() == null - && contactBase.getVoiceNumber() == null; - } - - boolean containsPii(ContactBase contactBase) { - return contactBase.getEmailAddress() != null - || contactBase.getFaxNumber() != null - || contactBase.getInternationalizedPostalInfo() != null - || contactBase.getLocalizedPostalInfo() != null - || contactBase.getVoiceNumber() != null; - } - - void assertAllPiiFieldsAreWipedOut(ImmutableList entities) { - ImmutableList.Builder notWipedEntities = new ImmutableList.Builder<>(); - for (ContactHistory entity : entities) { - if (!areAllPiiFieldsWiped(entity.getContactBase().get())) { - notWipedEntities.add(entity); - } - } - assertWithMessage("Not all PII fields of the contact history entities were wiped.") - .that(notWipedEntities.build()) - .isEmpty(); - } - - void assertAllEntitiesContainPii(ImmutableList entities) { - ImmutableList.Builder entitiesWithNoPii = new ImmutableList.Builder<>(); - for (ContactHistory entity : entities) { - if (!containsPii(entity.getContactBase().get())) { - entitiesWithNoPii.add(entity); - } - } - assertWithMessage("Not all contact history entities contain PII.") - .that(entitiesWithNoPii.build()) - .isEmpty(); + assertThat(response.getStatus()).isEqualTo(500); + assertThat(response.getPayload()).isEqualTo("Pipeline launch failed: cannot launch"); + verify(templates, times(1)) + .launch(eq("testProject"), eq("testRegion"), launchRequest.capture()); + assertThat(launchRequest.getValue().getLaunchParameter().getParameters()) + .containsExactlyEntriesIn(expectedParameters); } } diff --git a/core/src/test/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipelineTest.java b/core/src/test/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipelineTest.java new file mode 100644 index 000000000..32dea845e --- /dev/null +++ b/core/src/test/java/google/registry/beam/wipeout/WipeOutContactHistoryPiiPipelineTest.java @@ -0,0 +1,196 @@ +// Copyright 2023 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.beam.wipeout; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_CREATE; +import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ; +import static google.registry.testing.DatabaseHelper.loadAllOf; +import static google.registry.testing.DatabaseHelper.newContact; +import static google.registry.testing.DatabaseHelper.persistResource; +import static org.hibernate.cfg.AvailableSettings.ISOLATION; + +import com.google.common.collect.ImmutableList; +import google.registry.beam.TestPipelineExtension; +import google.registry.model.contact.Contact; +import google.registry.model.contact.ContactHistory; +import google.registry.model.contact.ContactPhoneNumber; +import google.registry.model.reporting.HistoryEntryDao; +import google.registry.persistence.transaction.JpaTestExtensions; +import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; +import google.registry.testing.FakeClock; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** Unit tests for {@link WipeOutContactHistoryPiiPipeline}. */ +public class WipeOutContactHistoryPiiPipelineTest { + + private static final int MIN_AGE_IN_MONTHS = 18; + private static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + + private final FakeClock clock = new FakeClock(DateTime.parse("2020-02-02T12:34:56Z")); + private final WipeOutContactHistoryPiiPipelineOptions options = + PipelineOptionsFactory.create().as(WipeOutContactHistoryPiiPipelineOptions.class); + private Contact contact1; + private Contact contact2; + + @RegisterExtension + final JpaIntegrationTestExtension jpa = + new JpaTestExtensions.Builder() + .withClock(clock) + .withProperty(ISOLATION, TRANSACTION_REPEATABLE_READ.name()) + .buildIntegrationTestExtension(); + + @RegisterExtension + final TestPipelineExtension pipeline = + TestPipelineExtension.create().enableAbandonedNodeEnforcement(true); + + @BeforeEach + void beforeEach() { + contact1 = + persistResource( + newContact("my-contact1") + .asBuilder() + .setEmailAddress("test@example.com") + .setFaxNumber( + new ContactPhoneNumber.Builder().setPhoneNumber("+12122122122").build()) + .build()); + contact2 = + persistResource( + newContact("my-contact2") + .asBuilder() + .setEmailAddress("test@example.tld") + .setVoiceNumber( + new ContactPhoneNumber.Builder().setPhoneNumber("+19177199177").build()) + .build()); + // T = 0 month; + persistResource(createHistory(contact1)); + // T = 5 months; + advanceMonths(5); + persistResource(createHistory(contact2)); + // T = 10 months; + advanceMonths(5); + persistResource(createHistory(contact1)); + persistResource(createHistory(contact2)); + // T = 20 months; + advanceMonths(10); + persistResource(createHistory(contact2)); + // T = 30 months; + advanceMonths(10); + options.setCutoffTime(DATE_TIME_FORMATTER.print(clock.nowUtc().minusMonths(MIN_AGE_IN_MONTHS))); + } + + @Test + void testSuccess() { + // Before the pipeline runs, every history entry should have an emali address. + assertThat( + loadAllOf(ContactHistory.class).stream() + .filter(e -> e.getContactBase().get().getEmailAddress() != null) + .count()) + .isEqualTo(5); + // Before the pipeline runs, contact history for contact1 should have fax numbers. + ImmutableList histories = + HistoryEntryDao.loadHistoryObjectsForResource(contact1.createVKey(), ContactHistory.class); + assertThat( + histories.stream().filter(e -> e.getContactBase().get().getFaxNumber() != null).count()) + .isEqualTo(2); + // Before the pipeline runs, contact history for contact2 should have voice numbers. + histories = + HistoryEntryDao.loadHistoryObjectsForResource(contact2.createVKey(), ContactHistory.class); + assertThat( + histories.stream() + .filter(e -> e.getContactBase().get().getVoiceNumber() != null) + .count()) + .isEqualTo(3); + WipeOutContactHistoryPiiPipeline wipeOutContactHistoryPiiPipeline = + new WipeOutContactHistoryPiiPipeline(options); + wipeOutContactHistoryPiiPipeline.run(pipeline).waitUntilFinish(); + histories = + HistoryEntryDao.loadHistoryObjectsForResource(contact1.createVKey(), ContactHistory.class); + assertThat(histories.size()).isEqualTo(2); + ImmutableList wipedEntries = + histories.stream() + .filter(e -> e.getContactBase().get().getEmailAddress() == null) + .collect(toImmutableList()); + // Only the history entry at T = 10 is wiped. The one at T = 10 is over 18 months old, but it + // is the most recent entry, so it is kept. + assertThat(wipedEntries.size()).isEqualTo(1); + assertThat(wipedEntries.get(0).getContactBase().get().getFaxNumber()).isNull(); + // With a new history entry at T = 30, the one at T = 10 is eligible for wipe out. Note the + // current time itself (therefore the cutoff time) has not changed. + persistResource(createHistory(contact1)); + wipeOutContactHistoryPiiPipeline.run(pipeline).waitUntilFinish(); + histories = + HistoryEntryDao.loadHistoryObjectsForResource(contact1.createVKey(), ContactHistory.class); + assertThat(histories.size()).isEqualTo(3); + wipedEntries = + histories.stream() + .filter(e -> e.getContactBase().get().getEmailAddress() == null) + .collect(toImmutableList()); + assertThat(wipedEntries.size()).isEqualTo(2); + // Check that the pipeline deals with multiple contacts correctly. + histories = + HistoryEntryDao.loadHistoryObjectsForResource(contact2.createVKey(), ContactHistory.class); + assertThat(histories.size()).isEqualTo(3); + wipedEntries = + histories.stream() + .filter(e -> e.getContactBase().get().getEmailAddress() == null) + .collect(toImmutableList()); + // Only the history entry at T = 10 is wiped. The one at T = 10 is over 18 months old, but it + // is the most recent entry, so it is kept. + assertThat(wipedEntries.size()).isEqualTo(2); + assertThat(wipedEntries.get(0).getContactBase().get().getVoiceNumber()).isNull(); + assertThat(wipedEntries.get(1).getContactBase().get().getVoiceNumber()).isNull(); + } + + @Test + void testSuccess_dryRun() { + options.setIsDryRun(true); + WipeOutContactHistoryPiiPipeline wipeOutContactHistoryPiiPipeline = + new WipeOutContactHistoryPiiPipeline(options); + wipeOutContactHistoryPiiPipeline.run(pipeline).waitUntilFinish(); + ImmutableList histories = + HistoryEntryDao.loadHistoryObjectsForResource(contact1.createVKey(), ContactHistory.class); + assertThat(histories.size()).isEqualTo(2); + assertThat( + histories.stream() + .filter(e -> e.getContactBase().get().getEmailAddress() == null) + .collect(toImmutableList())) + .isEmpty(); + } + + private ContactHistory createHistory(Contact contact) { + return new ContactHistory.Builder() + .setContact(contact) + .setType(CONTACT_CREATE) + .setRegistrarId("TheRegistrar") + .setModificationTime(clock.nowUtc()) + .build(); + } + + private void advanceMonths(int months) { + DateTime now = clock.nowUtc(); + DateTime next = now.plusMonths(months); + clock.advanceBy(new Duration(now, next)); + } +} diff --git a/release/cloudbuild-nomulus.yaml b/release/cloudbuild-nomulus.yaml index 9977191bf..4d8adac40 100644 --- a/release/cloudbuild-nomulus.yaml +++ b/release/cloudbuild-nomulus.yaml @@ -97,7 +97,9 @@ steps: google.registry.beam.rde.RdePipeline \ google/registry/beam/rde_pipeline_metadata.json \ google.registry.beam.resave.ResaveAllEppResourcesPipeline \ - google/registry/beam/resave_all_epp_resources_pipeline_metadata.json + google/registry/beam/resave_all_epp_resources_pipeline_metadata.json \ + google.registry.beam.wipeout.WipeOutContactHistoryPiiPipeline \ + google/registry/beam/wipe_out_contact_history_pii_pipeline_metadata.json # Tentatively build and publish Cloud SQL schema jar here, before schema release # process is finalized. Also publish nomulus:core jars that are needed for # server/schema compatibility tests.