From 222ccd912ac997ea7149a9aa08b21826dda68885 Mon Sep 17 00:00:00 2001 From: mcilwain Date: Fri, 7 Oct 2016 14:52:40 -0700 Subject: [PATCH] Centralize creation of MapreduceRunner in unit tests This also changes the default number of mapper shards in tests to 2, which is the number of EppResourceIndex buckets in unit tests. Running more shards than there are buckets causes unnecessary test load. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=135520601 --- .../google/registry/batch/DeleteProberDataActionTest.java | 4 +--- .../batch/ExpandRecurringBillingEventsActionTest.java | 3 +-- .../registry/export/ExportDomainListsActionTest.java | 4 +--- .../flows/async/DeleteContactsAndHostsActionTest.java | 4 +--- .../flows/async/DnsRefreshForHostRenameActionTest.java | 4 +--- .../flows/async/RefreshDnsOnHostRenameActionTest.java | 4 +--- .../google/registry/rde/RdeContactImportActionTest.java | 2 +- javatests/google/registry/rde/RdeStagingActionTest.java | 4 +--- javatests/google/registry/testing/mapreduce/BUILD | 2 ++ .../registry/testing/mapreduce/MapreduceTestCase.java | 8 ++++++++ .../tools/server/GenerateZoneFilesActionTest.java | 4 +--- .../tools/server/KillAllCommitLogsActionTest.java | 4 +--- .../tools/server/KillAllEppResourcesActionTest.java | 4 +--- .../tools/server/ResaveAllEppResourcesActionTest.java | 4 +--- 14 files changed, 22 insertions(+), 33 deletions(-) diff --git a/javatests/google/registry/batch/DeleteProberDataActionTest.java b/javatests/google/registry/batch/DeleteProberDataActionTest.java index e27faf3e4..1b5dcf986 100644 --- a/javatests/google/registry/batch/DeleteProberDataActionTest.java +++ b/javatests/google/registry/batch/DeleteProberDataActionTest.java @@ -23,10 +23,8 @@ import static google.registry.testing.DatastoreHelper.persistResource; import static google.registry.testing.DatastoreHelper.persistSimpleResource; import static google.registry.util.DateTimeUtils.START_OF_TIME; -import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.ImmutableObject; import google.registry.model.billing.BillingEvent; import google.registry.model.billing.BillingEvent.Reason; @@ -74,7 +72,7 @@ public class DeleteProberDataActionTest extends MapreduceTestCaseof(5), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.response = new FakeResponse(); action.isDryRun = false; } diff --git a/javatests/google/registry/batch/ExpandRecurringBillingEventsActionTest.java b/javatests/google/registry/batch/ExpandRecurringBillingEventsActionTest.java index db89d2d28..24d3f0678 100644 --- a/javatests/google/registry/batch/ExpandRecurringBillingEventsActionTest.java +++ b/javatests/google/registry/batch/ExpandRecurringBillingEventsActionTest.java @@ -32,7 +32,6 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Iterables; import com.googlecode.objectify.Key; import com.googlecode.objectify.VoidWork; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.billing.BillingEvent; import google.registry.model.billing.BillingEvent.Flag; import google.registry.model.billing.BillingEvent.Reason; @@ -71,7 +70,7 @@ public class ExpandRecurringBillingEventsActionTest @Before public void init() { action = new ExpandRecurringBillingEventsAction(); - action.mrRunner = new MapreduceRunner(Optional.of(5), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.clock = clock; action.cursorTimeParam = Optional.absent(); createTld("tld"); diff --git a/javatests/google/registry/export/ExportDomainListsActionTest.java b/javatests/google/registry/export/ExportDomainListsActionTest.java index 97e9cd170..14720650a 100644 --- a/javatests/google/registry/export/ExportDomainListsActionTest.java +++ b/javatests/google/registry/export/ExportDomainListsActionTest.java @@ -28,9 +28,7 @@ import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.ListOptions; import com.google.appengine.tools.cloudstorage.ListResult; -import com.google.common.base.Optional; import com.google.common.base.Splitter; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.registry.Registry; import google.registry.model.registry.Registry.TldType; import google.registry.testing.ExceptionRule; @@ -60,7 +58,7 @@ public class ExportDomainListsActionTest extends MapreduceTestCaseabsent(), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.response = new FakeResponse(); action.gcsBucket = "outputbucket"; action.gcsBufferSize = 500; diff --git a/javatests/google/registry/flows/async/DeleteContactsAndHostsActionTest.java b/javatests/google/registry/flows/async/DeleteContactsAndHostsActionTest.java index 6441d8966..7b6be957b 100644 --- a/javatests/google/registry/flows/async/DeleteContactsAndHostsActionTest.java +++ b/javatests/google/registry/flows/async/DeleteContactsAndHostsActionTest.java @@ -55,13 +55,11 @@ import static org.joda.time.Duration.standardSeconds; import com.google.appengine.api.taskqueue.QueueFactory; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TaskOptions.Method; -import com.google.common.base.Optional; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.googlecode.objectify.Key; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.EppResource; import google.registry.model.contact.ContactAddress; import google.registry.model.contact.ContactPhoneNumber; @@ -133,7 +131,7 @@ public class DeleteContactsAndHostsActionTest action = new DeleteContactsAndHostsAction(); action.clock = clock; - action.mrRunner = new MapreduceRunner(Optional.of(5), Optional.of(2)); + action.mrRunner = makeDefaultRunner(); action.response = new FakeResponse(); action.retrier = new Retrier(new FakeSleeper(clock), 1); action.queue = getQueue(QUEUE_ASYNC_DELETE); diff --git a/javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java b/javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java index 430447fbf..be03b203d 100644 --- a/javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java +++ b/javatests/google/registry/flows/async/DnsRefreshForHostRenameActionTest.java @@ -26,11 +26,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; -import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; import google.registry.dns.DnsQueue; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.host.HostResource; import google.registry.request.HttpException.BadRequestException; import google.registry.testing.ExceptionRule; @@ -64,7 +62,7 @@ public class DnsRefreshForHostRenameActionTest private void runMapreduce(String hostKeyString) throws Exception { action = new DnsRefreshForHostRenameAction(); action.hostKeyString = hostKeyString; - action.mrRunner = new MapreduceRunner(Optional.of(5), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.response = new FakeResponse(); action.run(); executeTasksUntilEmpty("mapreduce"); diff --git a/javatests/google/registry/flows/async/RefreshDnsOnHostRenameActionTest.java b/javatests/google/registry/flows/async/RefreshDnsOnHostRenameActionTest.java index 14e03ec5f..0b40badb6 100644 --- a/javatests/google/registry/flows/async/RefreshDnsOnHostRenameActionTest.java +++ b/javatests/google/registry/flows/async/RefreshDnsOnHostRenameActionTest.java @@ -33,10 +33,8 @@ import static org.joda.time.Duration.millis; import static org.joda.time.Duration.standardHours; import static org.joda.time.Duration.standardSeconds; -import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.host.HostResource; import google.registry.testing.ExceptionRule; import google.registry.testing.FakeClock; @@ -79,7 +77,7 @@ public class RefreshDnsOnHostRenameActionTest action = new RefreshDnsOnHostRenameAction(); action.clock = clock; - action.mrRunner = new MapreduceRunner(Optional.of(5), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.pullQueue = getQueue(QUEUE_ASYNC_HOST_RENAME); action.response = new FakeResponse(); action.retrier = new Retrier(new FakeSleeper(clock), 1); diff --git a/javatests/google/registry/rde/RdeContactImportActionTest.java b/javatests/google/registry/rde/RdeContactImportActionTest.java index d6577247d..d9d27f8ad 100644 --- a/javatests/google/registry/rde/RdeContactImportActionTest.java +++ b/javatests/google/registry/rde/RdeContactImportActionTest.java @@ -58,7 +58,7 @@ public class RdeContactImportActionTest extends MapreduceTestCaseabsent(), Optional.absent()); + mrRunner = makeDefaultRunner(); action = new RdeContactImportAction( mrRunner, response, diff --git a/javatests/google/registry/rde/RdeStagingActionTest.java b/javatests/google/registry/rde/RdeStagingActionTest.java index 3f0b5fffb..953f10994 100644 --- a/javatests/google/registry/rde/RdeStagingActionTest.java +++ b/javatests/google/registry/rde/RdeStagingActionTest.java @@ -36,14 +36,12 @@ import static java.util.Arrays.asList; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; import com.google.appengine.tools.cloudstorage.GcsServiceFactory; -import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import com.googlecode.objectify.VoidWork; import google.registry.keyring.api.Keyring; import google.registry.keyring.api.PgpHelper; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.common.Cursor; import google.registry.model.common.Cursor.CursorType; import google.registry.model.host.HostResource; @@ -123,7 +121,7 @@ public class RdeStagingActionTest extends MapreduceTestCase { inject.setStaticField(Ofy.class, "clock", clock); action = new RdeStagingAction(); action.clock = clock; - action.mrRunner = new MapreduceRunner(Optional.absent(), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.reducer = new RdeStagingReducer(); action.reducer.ghostrydeBufferSize = 31337; action.reducer.lockTimeout = Duration.standardHours(1); diff --git a/javatests/google/registry/testing/mapreduce/BUILD b/javatests/google/registry/testing/mapreduce/BUILD index 330ee3e44..799edcaff 100644 --- a/javatests/google/registry/testing/mapreduce/BUILD +++ b/javatests/google/registry/testing/mapreduce/BUILD @@ -26,6 +26,8 @@ java_library( "//third_party/java/mockito", "//third_party/java/servlet/servlet_api", "//third_party/java/truth", + "//java/google/registry/config", + "//java/google/registry/mapreduce", "//java/google/registry/model", "//java/google/registry/util", "//javatests/google/registry/testing", diff --git a/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java b/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java index a17bd90f6..d6a198cad 100644 --- a/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java +++ b/javatests/google/registry/testing/mapreduce/MapreduceTestCase.java @@ -32,6 +32,9 @@ import com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet; import com.google.appengine.tools.pipeline.impl.servlets.TaskHandler; import com.google.apphosting.api.ApiProxy; import com.google.common.base.CharMatcher; +import com.google.common.base.Optional; +import google.registry.config.RegistryEnvironment; +import google.registry.mapreduce.MapreduceRunner; import google.registry.testing.AppEngineRule; import google.registry.testing.FakeClock; import google.registry.testing.ShardableTestCase; @@ -84,6 +87,11 @@ public abstract class MapreduceTestCase extends ShardableTestCase { proxy.setProperty(LocalBlobstoreService.NO_STORAGE_PROPERTY, "true"); } + protected MapreduceRunner makeDefaultRunner() { + int numBuckets = RegistryEnvironment.get().config().getEppResourceIndexBucketCount(); + return new MapreduceRunner(Optional.of(numBuckets), Optional.of(1)); + } + protected List getTasks(String queueName) { return taskQueue.getQueueStateInfo().get(queueName).getTaskInfo(); } diff --git a/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java b/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java index e9e4b09a0..daadbee36 100644 --- a/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java +++ b/javatests/google/registry/tools/server/GenerateZoneFilesActionTest.java @@ -31,13 +31,11 @@ import static org.joda.time.Duration.standardDays; import com.google.appengine.tools.cloudstorage.GcsFilename; import com.google.appengine.tools.cloudstorage.GcsService; -import com.google.common.base.Optional; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.googlecode.objectify.Key; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.domain.secdns.DelegationSignerData; import google.registry.model.host.HostResource; import google.registry.testing.FakeClock; @@ -97,7 +95,7 @@ public class GenerateZoneFilesActionTest extends MapreduceTestCaseabsent(), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.bucket = "zonefiles-bucket"; action.gcsBufferSize = 123; action.datastoreRetention = standardDays(29); diff --git a/javatests/google/registry/tools/server/KillAllCommitLogsActionTest.java b/javatests/google/registry/tools/server/KillAllCommitLogsActionTest.java index 471e79f8f..10e03bf6d 100644 --- a/javatests/google/registry/tools/server/KillAllCommitLogsActionTest.java +++ b/javatests/google/registry/tools/server/KillAllCommitLogsActionTest.java @@ -27,12 +27,10 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME; import static java.util.Arrays.asList; import com.google.appengine.api.datastore.Entity; -import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.ImmutableObject; import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogCheckpoint; @@ -59,7 +57,7 @@ public class KillAllCommitLogsActionTest extends MapreduceTestCaseabsent(), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.response = new FakeResponse(); action.run(); executeTasksUntilEmpty("mapreduce"); diff --git a/javatests/google/registry/tools/server/KillAllEppResourcesActionTest.java b/javatests/google/registry/tools/server/KillAllEppResourcesActionTest.java index 770640eff..7e6b5261a 100644 --- a/javatests/google/registry/tools/server/KillAllEppResourcesActionTest.java +++ b/javatests/google/registry/tools/server/KillAllEppResourcesActionTest.java @@ -32,12 +32,10 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME; import static java.util.Arrays.asList; import com.google.appengine.api.datastore.Entity; -import com.google.common.base.Optional; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Iterables; import com.googlecode.objectify.Key; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.EppResource; import google.registry.model.ImmutableObject; import google.registry.model.billing.BillingEvent; @@ -84,7 +82,7 @@ public class KillAllEppResourcesActionTest extends MapreduceTestCaseabsent(), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.response = new FakeResponse(); action.run(); executeTasksUntilEmpty("mapreduce"); diff --git a/javatests/google/registry/tools/server/ResaveAllEppResourcesActionTest.java b/javatests/google/registry/tools/server/ResaveAllEppResourcesActionTest.java index e3de78f07..2cf93d3bd 100644 --- a/javatests/google/registry/tools/server/ResaveAllEppResourcesActionTest.java +++ b/javatests/google/registry/tools/server/ResaveAllEppResourcesActionTest.java @@ -18,8 +18,6 @@ import static com.google.common.truth.Truth.assertThat; import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.testing.DatastoreHelper.persistActiveContact; -import com.google.common.base.Optional; -import google.registry.mapreduce.MapreduceRunner; import google.registry.model.contact.ContactResource; import google.registry.testing.FakeResponse; import google.registry.testing.mapreduce.MapreduceTestCase; @@ -37,7 +35,7 @@ public class ResaveAllEppResourcesActionTest @Before public void init() { action = new ResaveAllEppResourcesAction(); - action.mrRunner = new MapreduceRunner(Optional.of(5), Optional.absent()); + action.mrRunner = makeDefaultRunner(); action.response = new FakeResponse(); }