zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-05-14 16:37:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

Make re-save environment entities command use batching

Browse source 
This makes it take a lot less time to run (roughly a 10X speedup).

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=161666391
This commit is contained in:
mcilwain 2017-07-12 08:11:51 -07:00 • committed by Ben McIlwain
parent b235565eef
commit 24587491c9
2 changed files with 20 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

23
java/google/registry/tools/ResaveEnvironmentEntitiesCommand.java
View file

@ -14,14 +14,13 @@
package google.registry.tools; package google.registry.tools;
import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Lists.partition;
import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.model.ofy.ObjectifyService.ofy;
import com.beust.jcommander.Parameters; import com.beust.jcommander.Parameters;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import com.googlecode.objectify.VoidWork; import com.googlecode.objectify.VoidWork;
import google.registry.model.ImmutableObject;
import google.registry.model.registrar.Registrar; import google.registry.model.registrar.Registrar;
import google.registry.model.registrar.RegistrarContact; import google.registry.model.registrar.RegistrarContact;
import google.registry.model.registry.Registry; import google.registry.model.registry.Registry;
@ -36,19 +35,25 @@ import google.registry.tools.Command.RemoteApiCommand;
@Parameters(commandDescription = "Re-save all environment entities.") @Parameters(commandDescription = "Re-save all environment entities.")
final class ResaveEnvironmentEntitiesCommand implements RemoteApiCommand { final class ResaveEnvironmentEntitiesCommand implements RemoteApiCommand {
private static final int BATCH_SIZE = 10;
@Override @Override
public void run() throws Exception { public void run() throws Exception {
Iterable<Key<? extends ImmutableObject>> keys = concat( batchSave(Registry.class);
ofy().load().type(Registrar.class).ancestor(getCrossTldKey()).keys(), batchSave(Registrar.class);
ofy().load().type(Registry.class).ancestor(getCrossTldKey()).keys(), batchSave(RegistrarContact.class);
ofy().load().type(RegistrarContact.class).ancestor(getCrossTldKey()).keys()); }
for (final Key<? extends ImmutableObject> key : keys) {
private static <T> void batchSave(Class<T> clazz) {
System.out.printf("Re-saving %s entities.\n", clazz.getSimpleName());
for (final Iterable<Key<T>> batch :
partition(ofy().load().type(clazz).ancestor(getCrossTldKey()).keys().list(), BATCH_SIZE)) {
ofy().transact(new VoidWork() { ofy().transact(new VoidWork() {
@Override @Override
public void vrun() { public void vrun() {
ofy().save().entity(ofy().load().key(key).now()); ofy().save().entities(ofy().load().keys(batch).values());
}}); }});
System.out.printf("Re-saved entity %s\n", key); System.out.printf("Re-saved entities batch: %s.\n", batch);
} }
} }
} }

6
javatests/google/registry/tools/ResaveEnvironmentEntitiesCommandTest.java
View file

@ -55,9 +55,9 @@ public class ResaveEnvironmentEntitiesCommandTest
assertThat(ofy().load().type(CommitLogMutation.class).keys()).isEmpty(); assertThat(ofy().load().type(CommitLogMutation.class).keys()).isEmpty();
runCommand(); runCommand();
// There are five entities that have been re-saved at this point (each in a separate // There are 5 entities that have been re-saved at this point (in 3 transactions, one for each
// transaction), so expect five manifests and five mutations. // type), so expect 3 manifests and 5 mutations.
assertThat(ofy().load().type(CommitLogManifest.class).keys()).hasSize(5); assertThat(ofy().load().type(CommitLogManifest.class).keys()).hasSize(3);
Iterable<ImmutableObject> savedEntities = Iterable<ImmutableObject> savedEntities =
transform( transform(
ofy().load().type(CommitLogMutation.class).list(), ofy().load().type(CommitLogMutation.class).list(),