From 07e2a7143311940db4bf98e2087814dd14f741f5 Mon Sep 17 00:00:00 2001 From: Weimin Yu Date: Tue, 22 Jun 2021 15:59:58 -0400 Subject: [PATCH] Fix appId during cross-project commitlog imports (#1213) * Fix appId during cross-project commitlog imports When importing commit logs from another project, we must override the appId in every entity key instances. The fixEntity method in the EntityImports class is a straightforward translation of the python function of the same name used by the storage team. --- .../google/registry/backup/BackupUtils.java | 9 +- .../registry/backup/CommitLogImports.java | 2 +- .../google/registry/backup/EntityImports.java | 115 ++++++++++++++++++ .../backup/RestoreCommitLogsAction.java | 2 +- .../registry/backup/EntityImportsTest.java | 87 +++++++++++++ .../backup/RestoreCommitLogsActionTest.java | 39 +++++- .../google/registry/backup/commitlog.data | Bin 0 -> 4067 bytes 7 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 core/src/main/java/google/registry/backup/EntityImports.java create mode 100644 core/src/test/java/google/registry/backup/EntityImportsTest.java create mode 100644 core/src/test/resources/google/registry/backup/commitlog.data diff --git a/core/src/main/java/google/registry/backup/BackupUtils.java b/core/src/main/java/google/registry/backup/BackupUtils.java index 6ccb882fa..8a036f319 100644 --- a/core/src/main/java/google/registry/backup/BackupUtils.java +++ b/core/src/main/java/google/registry/backup/BackupUtils.java @@ -56,12 +56,16 @@ public class BackupUtils { * *

The iterator reads from the stream on demand, and as such will fail if the stream is closed. */ - public static Iterator createDeserializingIterator(final InputStream input) { + public static Iterator createDeserializingIterator( + final InputStream input, boolean withAppIdOverride) { return new AbstractIterator() { @Override protected ImmutableObject computeNext() { EntityProto proto = new EntityProto(); if (proto.parseDelimitedFrom(input)) { // False means end of stream; other errors throw. + if (withAppIdOverride) { + proto = EntityImports.fixEntity(proto); + } return auditedOfy().load().fromEntity(EntityTranslator.createFromPb(proto)); } return endOfData(); @@ -70,6 +74,7 @@ public class BackupUtils { } public static ImmutableList deserializeEntities(byte[] bytes) { - return ImmutableList.copyOf(createDeserializingIterator(new ByteArrayInputStream(bytes))); + return ImmutableList.copyOf( + createDeserializingIterator(new ByteArrayInputStream(bytes), false)); } } diff --git a/core/src/main/java/google/registry/backup/CommitLogImports.java b/core/src/main/java/google/registry/backup/CommitLogImports.java index da7377f08..06429fc46 100644 --- a/core/src/main/java/google/registry/backup/CommitLogImports.java +++ b/core/src/main/java/google/registry/backup/CommitLogImports.java @@ -59,7 +59,7 @@ public final class CommitLogImports { InputStream inputStream) { try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment(); InputStream input = new BufferedInputStream(inputStream)) { - Iterator commitLogs = createDeserializingIterator(input); + Iterator commitLogs = createDeserializingIterator(input, false); checkState(commitLogs.hasNext()); checkState(commitLogs.next() instanceof CommitLogCheckpoint); diff --git a/core/src/main/java/google/registry/backup/EntityImports.java b/core/src/main/java/google/registry/backup/EntityImports.java new file mode 100644 index 000000000..1e87a3bef --- /dev/null +++ b/core/src/main/java/google/registry/backup/EntityImports.java @@ -0,0 +1,115 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.backup; + +import com.google.apphosting.api.ApiProxy; +import com.google.storage.onestore.v3.OnestoreEntity; +import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; +import com.google.storage.onestore.v3.OnestoreEntity.Path; +import com.google.storage.onestore.v3.OnestoreEntity.Property.Meaning; +import com.google.storage.onestore.v3.OnestoreEntity.PropertyValue.ReferenceValue; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +/** Utilities for handling imported Datastore entities. */ +public class EntityImports { + + /** + * Transitively sets the {@code appId} of all keys in a foreign entity to that of the current + * system. + */ + public static EntityProto fixEntity(EntityProto entityProto) { + String currentAappId = ApiProxy.getCurrentEnvironment().getAppId(); + if (Objects.equals(currentAappId, entityProto.getKey().getApp())) { + return entityProto; + } + return fixEntity(entityProto, currentAappId); + } + + private static EntityProto fixEntity(EntityProto entityProto, String appId) { + if (entityProto.hasKey()) { + fixKey(entityProto, appId); + } + + for (OnestoreEntity.Property property : entityProto.mutablePropertys()) { + fixProperty(property, appId); + } + + for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) { + fixProperty(property, appId); + } + + // CommitLogMutation embeds an entity as bytes, which needs additional fixes. + if (isCommitLogMutation(entityProto)) { + fixMutationEntityProtoBytes(entityProto, appId); + } + return entityProto; + } + + private static boolean isCommitLogMutation(EntityProto entityProto) { + if (!entityProto.hasKey()) { + return false; + } + Path path = entityProto.getKey().getPath(); + if (path.elementSize() == 0) { + return false; + } + return Objects.equals( + path.getElement(path.elementSize() - 1).getType(StandardCharsets.UTF_8), + "CommitLogMutation"); + } + + private static void fixMutationEntityProtoBytes(EntityProto entityProto, String appId) { + for (OnestoreEntity.Property property : entityProto.mutableRawPropertys()) { + if (Objects.equals(property.getName(), "entityProtoBytes")) { + OnestoreEntity.PropertyValue value = property.getValue(); + EntityProto fixedProto = + fixEntity(bytesToEntityProto(value.getStringValueAsBytes()), appId); + value.setStringValueAsBytes(fixedProto.toByteArray()); + return; + } + } + } + + private static void fixKey(EntityProto entityProto, String appId) { + entityProto.getMutableKey().setApp(appId); + } + + private static void fixKey(ReferenceValue referenceValue, String appId) { + referenceValue.setApp(appId); + } + + private static void fixProperty(OnestoreEntity.Property property, String appId) { + OnestoreEntity.PropertyValue value = property.getMutableValue(); + if (value.hasReferenceValue()) { + fixKey(value.getMutableReferenceValue(), appId); + return; + } + if (property.getMeaningEnum().equals(Meaning.ENTITY_PROTO)) { + EntityProto embeddedProto = bytesToEntityProto(value.getStringValueAsBytes()); + fixEntity(embeddedProto, appId); + value.setStringValueAsBytes(embeddedProto.toByteArray()); + } + } + + private static EntityProto bytesToEntityProto(byte[] bytes) { + EntityProto entityProto = new EntityProto(); + boolean isParsed = entityProto.parseFrom(bytes); + if (!isParsed) { + throw new IllegalStateException("Failed to parse raw bytes as EntityProto."); + } + return entityProto; + } +} diff --git a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java index 62b1c50d8..2f93667bd 100644 --- a/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java +++ b/core/src/main/java/google/registry/backup/RestoreCommitLogsAction.java @@ -104,7 +104,7 @@ public class RestoreCommitLogsAction implements Runnable { try (InputStream input = Channels.newInputStream( gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) { PeekingIterator commitLogs = - peekingIterator(createDeserializingIterator(input)); + peekingIterator(createDeserializingIterator(input, true)); lastCheckpoint = (CommitLogCheckpoint) commitLogs.next(); saveOfy(ImmutableList.of(lastCheckpoint)); // Save the checkpoint itself. while (commitLogs.hasNext()) { diff --git a/core/src/test/java/google/registry/backup/EntityImportsTest.java b/core/src/test/java/google/registry/backup/EntityImportsTest.java new file mode 100644 index 000000000..e5e105a3a --- /dev/null +++ b/core/src/test/java/google/registry/backup/EntityImportsTest.java @@ -0,0 +1,87 @@ +// Copyright 2021 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.backup; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.ofy.ObjectifyService.auditedOfy; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.EntityTranslator; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Resources; +import com.google.storage.onestore.v3.OnestoreEntity.EntityProto; +import google.registry.model.ofy.CommitLogCheckpoint; +import google.registry.testing.AppEngineExtension; +import google.registry.testing.DatastoreEntityExtension; +import java.io.InputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +public class EntityImportsTest { + + @RegisterExtension + @Order(value = 1) + final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); + + @RegisterExtension + final AppEngineExtension appEngine = + new AppEngineExtension.Builder().withDatastoreAndCloudSql().withoutCannedData().build(); + + private DatastoreService datastoreService; + + @BeforeEach + void beforeEach() { + datastoreService = DatastoreServiceFactory.getDatastoreService(); + } + + @Test + void importCommitLogs_keysFixed() throws Exception { + // Input resource is a standard commit log file whose entities has "AppId_1" as appId. The key + // fixes can be verified by checking that the appId of an imported entity's key has been updated + // to 'test' (which is set by AppEngineExtension) and/or that after persistence the imported + // entity can be loaded by Objectify. + try (InputStream commitLogInputStream = + Resources.getResource("google/registry/backup/commitlog.data").openStream()) { + ImmutableList entities = + loadEntityProtos(commitLogInputStream).stream() + .map(EntityImports::fixEntity) + .map(EntityTranslator::createFromPb) + .collect(ImmutableList.toImmutableList()); + // Verifies that the original appId has been overwritten. + assertThat(entities.get(0).getKey().getAppId()).isEqualTo("test"); + datastoreService.put(entities); + // Imported entity can be found by Ofy after appId conversion. + assertThat(auditedOfy().load().type(CommitLogCheckpoint.class).count()).isGreaterThan(0); + } + } + + private static ImmutableList loadEntityProtos(InputStream inputStream) { + ImmutableList.Builder protosBuilder = new ImmutableList.Builder<>(); + while (true) { + EntityProto proto = new EntityProto(); + boolean parsed = proto.parseDelimitedFrom(inputStream); + if (parsed && proto.isInitialized()) { + protosBuilder.add(proto); + } else { + break; + } + } + return protosBuilder.build(); + } +} diff --git a/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java b/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java index d935d7be4..b7001199d 100644 --- a/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java +++ b/core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java @@ -34,9 +34,11 @@ import com.google.appengine.tools.cloudstorage.GcsService; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.io.Resources; import com.google.common.primitives.Longs; import com.googlecode.objectify.Key; import google.registry.model.ImmutableObject; +import google.registry.model.domain.DomainBase; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; import google.registry.model.ofy.CommitLogCheckpointRoot; @@ -258,10 +260,40 @@ public class RestoreCommitLogsActionTest { assertInDatastore(CommitLogCheckpointRoot.create(now)); } + @Test + void testRestore_fromOtherProject() throws IOException { + // Input resource is a standard commit log file whose entities has "AppId_1" as appId. Among the + // entities are CommitLogMutations that have an embedded DomainBase and a ContactResource, both + // having "AppId_1" as appId. This test verifies that the embedded entities are properly + // imported, in particular, the domain's 'registrant' key can be used by Objectify to load the + // contact. + saveDiffFile( + gcsService, + Resources.toByteArray(Resources.getResource("google/registry/backup/commitlog.data")), + now); + action.run(); + auditedOfy().clearSessionCache(); + List domainBases = auditedOfy().load().type(DomainBase.class).list(); + assertThat(domainBases).hasSize(1); + DomainBase domainBase = domainBases.get(0); + // If the registrant is found, then the key instance in domainBase is fixed. + assertThat(auditedOfy().load().key(domainBase.getRegistrant().getOfyKey()).now()).isNotNull(); + } + static CommitLogCheckpoint createCheckpoint(DateTime now) { return CommitLogCheckpoint.create(now, toMap(getBucketIds(), x -> now)); } + static void saveDiffFile(GcsService gcsService, byte[] rawBytes, DateTime timestamp) + throws IOException { + gcsService.createOrReplace( + new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + timestamp), + new GcsFileOptions.Builder() + .addUserMetadata(LOWER_BOUND_CHECKPOINT, timestamp.minusMinutes(1).toString()) + .build(), + ByteBuffer.wrap(rawBytes)); + } + static Iterable saveDiffFile( GcsService gcsService, CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException { @@ -271,12 +303,7 @@ public class RestoreCommitLogsActionTest { for (ImmutableObject entity : allEntities) { serializeEntity(entity, output); } - gcsService.createOrReplace( - new GcsFilename(GCS_BUCKET, DIFF_FILE_PREFIX + now), - new GcsFileOptions.Builder() - .addUserMetadata(LOWER_BOUND_CHECKPOINT, now.minusMinutes(1).toString()) - .build(), - ByteBuffer.wrap(output.toByteArray())); + saveDiffFile(gcsService, output.toByteArray(), now); return allEntities; } diff --git a/core/src/test/resources/google/registry/backup/commitlog.data b/core/src/test/resources/google/registry/backup/commitlog.data new file mode 100644 index 0000000000000000000000000000000000000000..391e8f5a327c60d0cffdae9ef58ce69c930540eb GIT binary patch literal 4067 zcmd5;OK%%h6!xScabmY|9GZqx0wFw#qSn|k6^TFyKPIV*TlpwL_79lnukoXg@L99Sx50n*Kwk%n}x#QP3*sgdr`=7kS3}WD8QN^>cAnNUb)p_wR5$GsrS_5(4n-G+Y%aa`@T(ubxg2&OI1Q1i zqaG2vqBjY*m8Qj4q_v4)z_oy~b<(P!gf& zT+iFI=hf0V3}L>vxTBc*A-cJAza7A-baCjN-@=X0rM}^hq@kzASsvJPY{k~8xq3^f zX=(LF`sSUjjeBY%m6vOqTUE1UX*Y};YI8H8ZrwbtCU5JR{X(mnEV%i%O8idh1kcsW zclI?)O_o}uoRhWfdq+FBPr~jOq2L9G5YyJ}R*_Mgrdu}QD}l97L-28^@oGPYIEEm5 z=e8MjERo;U8I@7aui1u%CG5q1~gyvFonvPBJzu$+cg1GqSiyOowj z@N)_#OL8jjE@P_0DkEN$f(W=b6YS>Dy@K1zM6tFADXVp&IR=T@Ofk7GQ1Q;UDYF4r z7%|C7S~m>ctYs*-E4_0(R_!k%$3if(Th6W|60vyPeF>gs9ryE8J-WqqO#^>@hR~&1 z2#OnVabX^2G?Nz3Y$~1GA6J{zNq5=0B zTxLWgO-tam%@mas3Dc>TCuYsAqsL*GEmqdAZ?4DV@v{3YL>ZEuGBSum7r9pi5HVQB zCyogdUOGHeSIpXo^PA%Q0t8iuVVcwyKKK}%LrC3KnD^2@;$l) zWDJG#LOZnA;k6p2HRMG%i?Jrvh!N}DhzaL()rWD>Z^~VRCpRW!8PrOE((dNtvzokf^sRM*PR;h}yH-^1FhIc}TEV;U#s4M%|GY4-Kj9l%&&F__VN?Ca)?0y*> z(X2iT!Y9GTGkyOSL($nRmZ)x~6;4*ClGg;6!)y^t;%uYAS%Y1Lr9;OsTDKj=&<}N@ zwJ&$n^C!2$Fc&^VAwu%=bf@@ytv7S3U&R5gy4OmbN zBSTGFQEeVm+y5YMuMBjN&Vdr))L|-lAWf}bE8WUvZS8+yVGS1B>TZ`UYI4dlhK|#P z*ZdL2N}Y731ckX#2zNy{tJn(S6>u-XJj%K)Th^|niLTck1_Ib2B7S&mF+fP7UzOzT zT)9$86)NuX&@y_Aw7HH;m?xl$8y7jVlk$H=kHv1Rhu zv0^yl;n&Y3r8EDwhsOKT1a|a4PmTAbc9CIYUxFy|p`J4jY0Nsou4U2Afb<+hx|7ft z>{gIbb50K;Z*WF$eY3G3>G{6nN2G;r`}P49x68=0A+Nx^-@A6uXdR~44=GKjO0MWF z(CM|_)F$7vbmpJmeB5>~!^PfPol!zQMp}6IEDlV~_4vVlA->z(bF0a{ z*1`I&o^Isq-J{K$r4-KMzfO*jw}+GP{bb{XzPw%b-1L;9D+H?@9T=@~k5a9_P`WGv zvz#bQt#=eM@Yjm@KPf4InT}|Bm4wZ-^LaP%kIU@>1T~$j2IXx{_~+&K=o0%|brJE^ R#UxDJdh=8oN0w1#@dt~xFC72? literal 0 HcmV?d00001