Change commit log bucket counts in tests

I'm setting it to three buckets across all tests, because the default one bucket
wasn't realistic enough, and allowed some tests to pass that shouldn't have,
essentially by accident.

This also changes RegistryConfig from being an interface to being an abstract
base class. The medium term goal here is to have it be a static class so that it
can provide fields from the YAML-derived POJO in situations where Dagger
injection isn't feasible.

The expected end state is as follows:

default-config.yaml -- The master config file that provides defaults for all
values.

nomulus-config.yaml -- A per-environment config file that overrides the defaults
from the previous file.

YamlConfig.java -- The POJO that the aforementioned YAML files are deserialized
into.

RegistryConfig.java -- Contains a static, memoized instance of YamlConfig and
provides static methods for getting some of those values.

ConfigModule -- Will become a static inner class of RegistryConfig, using Dagger
to provide most of the fields from the memoized YamlConfig instance. This way,
all configuration will be coming from a single place: RegistryConfig.java.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=143567288
This commit is contained in:
mcilwain 2017-01-04 10:18:52 -08:00 committed by Ben McIlwain
parent 734130aa73
commit c35c3a678b
18 changed files with 119 additions and 206 deletions

View file

@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.base.Verify.verify; import static com.google.common.base.Verify.verify;
import static google.registry.bigquery.BigqueryUtils.toJobReferenceString; import static google.registry.bigquery.BigqueryUtils.toJobReferenceString;
import static google.registry.config.RegistryConfig.getProjectId;
import static org.joda.time.DateTimeZone.UTC; import static org.joda.time.DateTimeZone.UTC;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
@ -60,7 +61,6 @@ import google.registry.bigquery.BigqueryUtils.DestinationFormat;
import google.registry.bigquery.BigqueryUtils.SourceFormat; import google.registry.bigquery.BigqueryUtils.SourceFormat;
import google.registry.bigquery.BigqueryUtils.TableType; import google.registry.bigquery.BigqueryUtils.TableType;
import google.registry.bigquery.BigqueryUtils.WriteDisposition; import google.registry.bigquery.BigqueryUtils.WriteDisposition;
import google.registry.config.RegistryEnvironment;
import google.registry.util.FormattingLogger; import google.registry.util.FormattingLogger;
import google.registry.util.NonFinalForTesting; import google.registry.util.NonFinalForTesting;
import google.registry.util.Sleeper; import google.registry.util.Sleeper;
@ -246,7 +246,7 @@ public class BigqueryConnection implements AutoCloseable {
} }
public DestinationTable build() { public DestinationTable build() {
tableRef.setProjectId(getEnvironmentProjectId()); tableRef.setProjectId(getProjectId());
table.setTableReference(tableRef); table.setTableReference(tableRef);
checkState(!isNullOrEmpty(table.getTableReference().getDatasetId())); checkState(!isNullOrEmpty(table.getTableReference().getDatasetId()));
checkState(!isNullOrEmpty(table.getTableReference().getTableId())); checkState(!isNullOrEmpty(table.getTableReference().getTableId()));
@ -704,16 +704,6 @@ public class BigqueryConnection implements AutoCloseable {
} }
} }
/** Returns the projectId set by the environment, or {@code null} if none is set. */
public static String getEnvironmentProjectId() {
return RegistryEnvironment.get().config().getProjectId();
}
/** Returns the projectId associated with this bigquery connection. */
public String getProjectId() {
return getEnvironmentProjectId();
}
/** Returns the dataset name that this bigquery connection uses by default. */ /** Returns the dataset name that this bigquery connection uses by default. */
public String getDatasetId() { public String getDatasetId() {
return datasetId; return datasetId;

View file

@ -79,8 +79,8 @@ public final class ConfigModule {
@Provides @Provides
@Config("projectId") @Config("projectId")
public static String provideProjectId(RegistryConfig config) { public static String provideProjectId() {
return config.getProjectId(); return RegistryConfig.getProjectId();
} }
/** /**

View file

@ -19,7 +19,6 @@ import static google.registry.config.ConfigUtils.makeUrl;
import static org.joda.time.Duration.standardDays; import static org.joda.time.Duration.standardDays;
import com.google.appengine.api.utils.SystemProperty; import com.google.appengine.api.utils.SystemProperty;
import com.google.common.base.Ascii;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.google.common.net.HostAndPort; import com.google.common.net.HostAndPort;
import java.net.URL; import java.net.URL;
@ -34,7 +33,7 @@ import org.joda.time.Duration;
* described in the {@link RegistryConfigLoader} documentation. * described in the {@link RegistryConfigLoader} documentation.
*/ */
@Immutable @Immutable
public final class ProductionRegistryConfigExample implements RegistryConfig { public final class ProductionRegistryConfigExample extends RegistryConfig {
private final RegistryEnvironment environment; private final RegistryEnvironment environment;
@ -49,22 +48,6 @@ public final class ProductionRegistryConfigExample implements RegistryConfig {
this.environment = checkNotNull(environment); this.environment = checkNotNull(environment);
} }
@Override
public String getProjectId() {
String prodProjectId = "domain-registry";
switch (environment) {
case PRODUCTION:
return prodProjectId;
default:
return prodProjectId + "-" + Ascii.toLowerCase(environment.name());
}
}
@Override
public int getCommitLogBucketCount() {
return 100; // if you decrease this number, the world ends
}
/** /**
* {@inheritDoc} * {@inheritDoc}
* *
@ -79,7 +62,7 @@ public final class ProductionRegistryConfigExample implements RegistryConfig {
@Override @Override
public String getSnapshotsBucket() { public String getSnapshotsBucket() {
return getProjectId() + "-snapshots"; return RegistryConfig.getProjectId() + "-snapshots";
} }
@Override @Override

View file

@ -14,6 +14,7 @@
package google.registry.config; package google.registry.config;
import com.google.common.base.Ascii;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.google.common.net.HostAndPort; import com.google.common.net.HostAndPort;
import java.net.URL; import java.net.URL;
@ -24,22 +25,31 @@ import org.joda.time.Duration;
* *
* <p>The goal of this custom configuration system is to have our project environments configured * <p>The goal of this custom configuration system is to have our project environments configured
* in type-safe Java code that can be refactored, rather than XML files and system properties. * in type-safe Java code that can be refactored, rather than XML files and system properties.
*
* <p><b>Note:</b> This interface is deprecated by {@link ConfigModule}.
*/ */
public interface RegistryConfig { public abstract class RegistryConfig {
/** /**
* Returns the App Engine project ID, which is based off the environment name. * Returns the App Engine project ID, which is based off the environment name.
*/ */
public String getProjectId(); public static String getProjectId() {
String prodProjectId = "domain-registry";
RegistryEnvironment environment = RegistryEnvironment.get();
switch (environment) {
case PRODUCTION:
case UNITTEST:
case LOCAL:
return prodProjectId;
default:
return prodProjectId + "-" + Ascii.toLowerCase(environment.name());
}
}
/** /**
* Returns the Google Cloud Storage bucket for storing backup snapshots. * Returns the Google Cloud Storage bucket for storing backup snapshots.
* *
* @see google.registry.export.ExportSnapshotServlet * @see google.registry.export.ExportSnapshotServlet
*/ */
public String getSnapshotsBucket(); public abstract String getSnapshotsBucket();
/** /**
* Number of sharded commit log buckets. * Number of sharded commit log buckets.
@ -54,7 +64,14 @@ public interface RegistryConfig {
* *
* @see google.registry.model.ofy.CommitLogBucket * @see google.registry.model.ofy.CommitLogBucket
*/ */
public int getCommitLogBucketCount(); public static int getCommitLogBucketCount() {
switch (RegistryEnvironment.get()) {
case UNITTEST:
return 3;
default:
return 100;
}
}
/** /**
* Returns the length of time before commit logs should be deleted from datastore. * Returns the length of time before commit logs should be deleted from datastore.
@ -65,26 +82,26 @@ public interface RegistryConfig {
* @see google.registry.backup.DeleteOldCommitLogsAction * @see google.registry.backup.DeleteOldCommitLogsAction
* @see google.registry.model.translators.CommitLogRevisionsTranslatorFactory * @see google.registry.model.translators.CommitLogRevisionsTranslatorFactory
*/ */
public Duration getCommitLogDatastoreRetention(); public abstract Duration getCommitLogDatastoreRetention();
/** /**
* Returns {@code true} if TMCH certificate authority should be in testing mode. * Returns {@code true} if TMCH certificate authority should be in testing mode.
* *
* @see google.registry.tmch.TmchCertificateAuthority * @see google.registry.tmch.TmchCertificateAuthority
*/ */
public boolean getTmchCaTestingMode(); public abstract boolean getTmchCaTestingMode();
public Optional<String> getECatcherAddress(); public abstract Optional<String> getECatcherAddress();
/** /**
* Returns the address of the Nomulus app HTTP server. * Returns the address of the Nomulus app HTTP server.
* *
* <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API. * <p>This is used by the {@code nomulus} tool to connect to the App Engine remote API.
*/ */
public HostAndPort getServer(); public abstract HostAndPort getServer();
/** Returns the amount of time a singleton should be cached, before expiring. */ /** Returns the amount of time a singleton should be cached, before expiring. */
public Duration getSingletonCacheRefreshDuration(); public abstract Duration getSingletonCacheRefreshDuration();
/** /**
* Returns the amount of time a domain label list should be cached in memory before expiring. * Returns the amount of time a domain label list should be cached in memory before expiring.
@ -92,31 +109,31 @@ public interface RegistryConfig {
* @see google.registry.model.registry.label.ReservedList * @see google.registry.model.registry.label.ReservedList
* @see google.registry.model.registry.label.PremiumList * @see google.registry.model.registry.label.PremiumList
*/ */
public Duration getDomainLabelListCacheDuration(); public abstract Duration getDomainLabelListCacheDuration();
/** Returns the amount of time a singleton should be cached in persist mode, before expiring. */ /** Returns the amount of time a singleton should be cached in persist mode, before expiring. */
public Duration getSingletonCachePersistDuration(); public abstract Duration getSingletonCachePersistDuration();
/** /**
* Returns the header text at the top of the reserved terms exported list. * Returns the header text at the top of the reserved terms exported list.
* *
* @see google.registry.export.ExportUtils#exportReservedTerms * @see google.registry.export.ExportUtils#exportReservedTerms
*/ */
public String getReservedTermsExportDisclaimer(); public abstract String getReservedTermsExportDisclaimer();
/** /**
* Returns a display name that is used on outgoing emails sent by Nomulus. * Returns a display name that is used on outgoing emails sent by Nomulus.
* *
* @see google.registry.util.SendEmailUtils * @see google.registry.util.SendEmailUtils
*/ */
public String getGoogleAppsAdminEmailDisplayName(); public abstract String getGoogleAppsAdminEmailDisplayName();
/** /**
* Returns the email address that outgoing emails from the app are sent from. * Returns the email address that outgoing emails from the app are sent from.
* *
* @see google.registry.util.SendEmailUtils * @see google.registry.util.SendEmailUtils
*/ */
public String getGoogleAppsSendFromEmailAddress(); public abstract String getGoogleAppsSendFromEmailAddress();
/** /**
* Returns default WHOIS server to use when {@code Registrar#getWhoisServer()} is {@code null}. * Returns default WHOIS server to use when {@code Registrar#getWhoisServer()} is {@code null}.
@ -124,33 +141,33 @@ public interface RegistryConfig {
* @see "google.registry.whois.DomainWhoisResponse" * @see "google.registry.whois.DomainWhoisResponse"
* @see "google.registry.whois.RegistrarWhoisResponse" * @see "google.registry.whois.RegistrarWhoisResponse"
*/ */
public String getRegistrarDefaultWhoisServer(); public abstract String getRegistrarDefaultWhoisServer();
/** /**
* Returns the default referral URL that is used unless registrars have specified otherwise. * Returns the default referral URL that is used unless registrars have specified otherwise.
*/ */
public URL getRegistrarDefaultReferralUrl(); public abstract URL getRegistrarDefaultReferralUrl();
/** /**
* Returns the number of EppResourceIndex buckets to be used. * Returns the number of EppResourceIndex buckets to be used.
*/ */
public int getEppResourceIndexBucketCount(); public abstract int getEppResourceIndexBucketCount();
/** /**
* Returns the base duration that gets doubled on each retry within {@code Ofy}. * Returns the base duration that gets doubled on each retry within {@code Ofy}.
*/ */
public Duration getBaseOfyRetryDuration(); public abstract Duration getBaseOfyRetryDuration();
/** /**
* Returns the global automatic transfer length for contacts. After this amount of time has * Returns the global automatic transfer length for contacts. After this amount of time has
* elapsed, the transfer is automatically approved. * elapsed, the transfer is automatically approved.
*/ */
public Duration getContactAutomaticTransferLength(); public abstract Duration getContactAutomaticTransferLength();
/** /**
* Returns the clientId of the registrar used by the {@code CheckApiServlet}. * Returns the clientId of the registrar used by the {@code CheckApiServlet}.
*/ */
public String getCheckApiServletRegistrarClientId(); public abstract String getCheckApiServletRegistrarClientId();
// XXX: Please consider using ConfigModule instead of adding new methods to this file. // XXX: Please consider using ConfigModule instead of adding new methods to this file.
} }

View file

@ -25,20 +25,10 @@ import org.joda.time.Duration;
/** /**
* An implementation of RegistryConfig for unit testing that contains suitable testing data. * An implementation of RegistryConfig for unit testing that contains suitable testing data.
*/ */
public class TestRegistryConfig implements RegistryConfig { public class TestRegistryConfig extends RegistryConfig {
public TestRegistryConfig() {} public TestRegistryConfig() {}
@Override
public String getProjectId() {
return "domain-registry";
}
@Override
public int getCommitLogBucketCount() {
return 1;
}
@Override @Override
public Duration getCommitLogDatastoreRetention() { public Duration getCommitLogDatastoreRetention() {
return Duration.standardDays(30); return Duration.standardDays(30);
@ -46,7 +36,7 @@ public class TestRegistryConfig implements RegistryConfig {
@Override @Override
public String getSnapshotsBucket() { public String getSnapshotsBucket() {
return getProjectId() + "-snapshots"; return RegistryConfig.getProjectId() + "-snapshots";
} }
@Override @Override

View file

@ -17,6 +17,7 @@ package google.registry.model.ofy;
import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.DiscreteDomain.integers; import static com.google.common.collect.DiscreteDomain.integers;
import static com.googlecode.objectify.ObjectifyService.ofy; import static com.googlecode.objectify.ObjectifyService.ofy;
import static google.registry.config.RegistryConfig.getCommitLogBucketCount;
import static google.registry.util.DateTimeUtils.START_OF_TIME; import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.common.base.Function; import com.google.common.base.Function;
@ -29,7 +30,7 @@ import com.google.common.collect.Range;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.Id; import com.googlecode.objectify.annotation.Id;
import google.registry.config.RegistryEnvironment; import google.registry.config.RegistryConfig;
import google.registry.model.Buildable; import google.registry.model.Buildable;
import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObject;
import google.registry.model.annotations.NotBackedUp; import google.registry.model.annotations.NotBackedUp;
@ -41,8 +42,8 @@ import org.joda.time.DateTime;
/** /**
* Root for a random commit log bucket. * Root for a random commit log bucket.
* *
* <p>This is used to shard {@link CommitLogManifest} objects into {@link * <p>This is used to shard {@link CommitLogManifest} objects into
* google.registry.config.RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases * {@link RegistryConfig#getCommitLogBucketCount() N} entity groups. This increases
* transaction throughput, while maintaining the ability to perform strongly-consistent ancestor * transaction throughput, while maintaining the ability to perform strongly-consistent ancestor
* queries. * queries.
* *
@ -53,11 +54,11 @@ import org.joda.time.DateTime;
@NotBackedUp(reason = Reason.COMMIT_LOGS) @NotBackedUp(reason = Reason.COMMIT_LOGS)
public class CommitLogBucket extends ImmutableObject implements Buildable { public class CommitLogBucket extends ImmutableObject implements Buildable {
private static final RegistryEnvironment ENVIRONMENT = RegistryEnvironment.get(); /**
* Ranges from 1 to {@link RegistryConfig#getCommitLogBucketCount()}, inclusive; starts at 1 since
/** Ranges from 1 to {@link #getNumBuckets()}, inclusive; starts at 1 since IDs can't be 0. */ * IDs can't be 0.
@Id */
long bucketNum; @Id long bucketNum;
/** The timestamp of the last {@link CommitLogManifest} written to this bucket. */ /** The timestamp of the last {@link CommitLogManifest} written to this bucket. */
DateTime lastWrittenTime = START_OF_TIME; DateTime lastWrittenTime = START_OF_TIME;
@ -90,12 +91,8 @@ public class CommitLogBucket extends ImmutableObject implements Buildable {
return ContiguousSet.create(getBucketIdRange(), integers()); return ContiguousSet.create(getBucketIdRange(), integers());
} }
private static int getNumBuckets() {
return ENVIRONMENT.config().getCommitLogBucketCount();
}
private static Range<Integer> getBucketIdRange() { private static Range<Integer> getBucketIdRange() {
return Range.closed(1, getNumBuckets()); return Range.closed(1, getCommitLogBucketCount());
} }
/** Returns an arbitrary numeric bucket ID. Default behavior is randomly chosen IDs. */ /** Returns an arbitrary numeric bucket ID. Default behavior is randomly chosen IDs. */
@ -116,7 +113,7 @@ public class CommitLogBucket extends ImmutableObject implements Buildable {
@Override @Override
public Integer get() { public Integer get() {
return random.nextInt(getNumBuckets()) + 1; // Add 1 since IDs can't be 0. return random.nextInt(getCommitLogBucketCount()) + 1; // Add 1 since IDs can't be 0.
} }
}; };

View file

@ -62,8 +62,10 @@ public class CommitLogCheckpointActionTest {
task.clock = new FakeClock(now); task.clock = new FakeClock(now);
task.strategy = strategy; task.strategy = strategy;
task.taskEnqueuer = new TaskEnqueuer(new Retrier(null, 1)); task.taskEnqueuer = new TaskEnqueuer(new Retrier(null, 1));
when(strategy.computeCheckpoint()).thenReturn( when(strategy.computeCheckpoint())
CommitLogCheckpoint.create(now, ImmutableMap.of(1, START_OF_TIME))); .thenReturn(
CommitLogCheckpoint.create(
now, ImmutableMap.of(1, START_OF_TIME, 2, START_OF_TIME, 3, START_OF_TIME)));
} }
@Test @Test

View file

@ -24,7 +24,6 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.googlecode.objectify.VoidWork; import com.googlecode.objectify.VoidWork;
import google.registry.config.TestRegistryConfig;
import google.registry.model.common.Cursor; import google.registry.model.common.Cursor;
import google.registry.model.common.Cursor.CursorType; import google.registry.model.common.Cursor.CursorType;
import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogBucket;
@ -93,13 +92,6 @@ public class CommitLogCheckpointStrategyTest {
strategy.clock = clock; strategy.clock = clock;
strategy.ofy = ofy; strategy.ofy = ofy;
// Use three commit log buckets for easier but sufficiently complex testing.
configRule.override(new TestRegistryConfig() {
@Override
public int getCommitLogBucketCount() {
return 3;
}});
// Need to inject clock into Ofy so that createTld() below will get the right time. // Need to inject clock into Ofy so that createTld() below will get the right time.
inject.setStaticField(Ofy.class, "clock", clock); inject.setStaticField(Ofy.class, "clock", clock);
// Inject a fake bucket ID supplier so we can dole out specific bucket IDs to commit logs. // Inject a fake bucket ID supplier so we can dole out specific bucket IDs to commit logs.

View file

@ -15,10 +15,11 @@
package google.registry.backup; package google.registry.backup;
import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertThat;
import static google.registry.config.RegistryConfig.getCommitLogBucketCount;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static org.joda.time.Duration.millis; import static org.joda.time.Duration.millis;
import com.googlecode.objectify.VoidWork; import com.googlecode.objectify.VoidWork;
import google.registry.config.TestRegistryConfig;
import google.registry.model.ofy.CommitLogManifest; import google.registry.model.ofy.CommitLogManifest;
import google.registry.model.ofy.CommitLogMutation; import google.registry.model.ofy.CommitLogMutation;
import google.registry.model.ofy.Ofy; import google.registry.model.ofy.Ofy;
@ -26,10 +27,8 @@ import google.registry.model.registrar.Registrar;
import google.registry.testing.AppEngineRule; import google.registry.testing.AppEngineRule;
import google.registry.testing.ExceptionRule; import google.registry.testing.ExceptionRule;
import google.registry.testing.FakeClock; import google.registry.testing.FakeClock;
import google.registry.testing.RegistryConfigRule;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.Duration; import org.joda.time.Duration;
import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
@ -44,29 +43,28 @@ public class DeleteOldCommitLogsActionTest {
.withDatastore() .withDatastore()
.build(); .build();
@Rule
public final RegistryConfigRule configRule = new RegistryConfigRule();
@Rule @Rule
public final ExceptionRule thrown = new ExceptionRule(); public final ExceptionRule thrown = new ExceptionRule();
private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ")); private final FakeClock clock = new FakeClock(DateTime.parse("2000-01-01TZ"));
private final Ofy ofy = new Ofy(clock); private final Ofy ofy = new Ofy(clock);
private final DeleteOldCommitLogsAction task = new DeleteOldCommitLogsAction();
@Before private void runInAllBuckets(int maxDeletes) {
public void before() throws Exception { for (int bucketNum = 1; bucketNum <= getCommitLogBucketCount(); bucketNum++) {
task.bucketNum = 1; DeleteOldCommitLogsAction task = new DeleteOldCommitLogsAction();
task.clock = clock; task.bucketNum = bucketNum;
task.maxAge = Duration.millis(2); task.clock = clock;
task.maxDeletes = 4; task.maxAge = Duration.millis(2);
task.ofy = ofy; task.maxDeletes = maxDeletes;
task.ofy = ofy;
task.run();
}
} }
@Test @Test
public void testRun_noCommitLogs_doesNothing() throws Exception { public void testRun_noCommitLogs_doesNothing() throws Exception {
assertManifestAndMutationCounts(0, 0); assertManifestAndMutationCounts(0, 0);
task.run(); runInAllBuckets(4);
assertManifestAndMutationCounts(0, 0); assertManifestAndMutationCounts(0, 0);
} }
@ -75,7 +73,7 @@ public class DeleteOldCommitLogsActionTest {
createCommitLog(); createCommitLog();
clock.advanceOneMilli(); clock.advanceOneMilli();
assertManifestAndMutationCounts(1, 2); assertManifestAndMutationCounts(1, 2);
task.run(); runInAllBuckets(4);
assertManifestAndMutationCounts(1, 2); assertManifestAndMutationCounts(1, 2);
} }
@ -83,7 +81,7 @@ public class DeleteOldCommitLogsActionTest {
public void testRun_commitLogEqualToThreshold_doesntGetDeleted() throws Exception { public void testRun_commitLogEqualToThreshold_doesntGetDeleted() throws Exception {
createCommitLog(); createCommitLog();
clock.advanceBy(millis(2)); clock.advanceBy(millis(2));
task.run(); runInAllBuckets(4);
assertManifestAndMutationCounts(1, 2); assertManifestAndMutationCounts(1, 2);
} }
@ -91,7 +89,7 @@ public class DeleteOldCommitLogsActionTest {
public void testRun_commitLogOlderThanThreshold_getsDeleted() throws Exception { public void testRun_commitLogOlderThanThreshold_getsDeleted() throws Exception {
createCommitLog(); createCommitLog();
clock.advanceBy(millis(3)); clock.advanceBy(millis(3));
task.run(); runInAllBuckets(4);
assertManifestAndMutationCounts(0, 0); assertManifestAndMutationCounts(0, 0);
} }
@ -101,34 +99,31 @@ public class DeleteOldCommitLogsActionTest {
clock.advanceBy(millis(3)); clock.advanceBy(millis(3));
createCommitLog(); createCommitLog();
assertManifestAndMutationCounts(2, 4); assertManifestAndMutationCounts(2, 4);
task.run(); runInAllBuckets(4);
assertManifestAndMutationCounts(1, 2); assertManifestAndMutationCounts(1, 2);
} }
@Test @Test
public void testRun_twoOlderThanThreshold_bothGetDeletedInSameTransaction() throws Exception { public void testRun_twoOlderThanThreshold_bothGetDeletedInSameTransaction() throws Exception {
task.maxDeletes = 2;
createCommitLog(); createCommitLog();
clock.advanceOneMilli(); clock.advanceOneMilli();
createCommitLog(); createCommitLog();
clock.advanceBy(millis(3)); clock.advanceBy(millis(3));
assertManifestAndMutationCounts(2, 4); assertManifestAndMutationCounts(2, 4);
task.run(); runInAllBuckets(2);
assertManifestAndMutationCounts(0, 0); assertManifestAndMutationCounts(0, 0);
} }
@Test @Test
public void testRun_twoOlderThanThreshold_bothGetDeletedInTwoTransactions() throws Exception { public void testRun_twoOlderThanThreshold_bothGetDeletedInTwoTransactions() throws Exception {
task.maxDeletes = 1;
createCommitLog(); createCommitLog();
clock.advanceOneMilli(); clock.advanceOneMilli();
createCommitLog(); createCommitLog();
clock.advanceBy(millis(3)); clock.advanceBy(millis(3));
createCommitLog(); createCommitLog();
assertManifestAndMutationCounts(3, 6); assertManifestAndMutationCounts(3, 6);
task.run(); runInAllBuckets(1);
assertManifestAndMutationCounts(2, 4); runInAllBuckets(1);
task.run();
assertManifestAndMutationCounts(1, 2); assertManifestAndMutationCounts(1, 2);
} }
@ -136,20 +131,22 @@ public class DeleteOldCommitLogsActionTest {
public void testRun_commitLogOlderButInADifferentBucket_doesntGetDeleted() throws Exception { public void testRun_commitLogOlderButInADifferentBucket_doesntGetDeleted() throws Exception {
createCommitLog(); createCommitLog();
clock.advanceBy(millis(31337)); clock.advanceBy(millis(31337));
configRule.override(new TestRegistryConfig() { int usedBucketNum = ofy().load().type(CommitLogManifest.class).list().get(0).getBucketId();
@Override public int getCommitLogBucketCount() { return 2; } DeleteOldCommitLogsAction task = new DeleteOldCommitLogsAction();
}); task.bucketNum = (usedBucketNum % getCommitLogBucketCount()) + 1;
task.bucketNum = 2; task.clock = clock;
task.maxAge = Duration.millis(2);
task.maxDeletes = 20;
task.ofy = ofy;
task.run(); task.run();
assertManifestAndMutationCounts(1, 2); assertManifestAndMutationCounts(1, 2);
} }
@Test @Test
public void testRun_lessThanATenthOfOldData_doesntGetDeleted() throws Exception { public void testRun_lessThanATenthOfOldData_doesntGetDeleted() throws Exception {
task.maxDeletes = 20;
createCommitLog(); createCommitLog();
clock.advanceBy(millis(2)); clock.advanceBy(millis(2));
task.run(); runInAllBuckets(20);
assertManifestAndMutationCounts(1, 2); assertManifestAndMutationCounts(1, 2);
} }

View file

@ -30,7 +30,6 @@ import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import com.googlecode.objectify.ObjectifyService; import com.googlecode.objectify.ObjectifyService;
import google.registry.config.TestRegistryConfig;
import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObject;
import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogBucket;
import google.registry.model.ofy.CommitLogCheckpoint; import google.registry.model.ofy.CommitLogCheckpoint;
@ -38,7 +37,6 @@ import google.registry.model.ofy.CommitLogManifest;
import google.registry.model.ofy.CommitLogMutation; import google.registry.model.ofy.CommitLogMutation;
import google.registry.testing.AppEngineRule; import google.registry.testing.AppEngineRule;
import google.registry.testing.GcsTestingUtils; import google.registry.testing.GcsTestingUtils;
import google.registry.testing.RegistryConfigRule;
import google.registry.testing.TestObject; import google.registry.testing.TestObject;
import java.util.List; import java.util.List;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@ -52,20 +50,11 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class) @RunWith(JUnit4.class)
public class ExportCommitLogDiffActionTest { public class ExportCommitLogDiffActionTest {
private static final int NUM_BUCKETS = 3;
@Rule @Rule
public final AppEngineRule appEngine = AppEngineRule.builder() public final AppEngineRule appEngine = AppEngineRule.builder()
.withDatastore() .withDatastore()
.build(); .build();
@Rule
public final RegistryConfigRule configRule = new RegistryConfigRule(
new TestRegistryConfig() {
@Override public int getCommitLogBucketCount() {
return NUM_BUCKETS;
}});
/** Local GCS service available for testing. */ /** Local GCS service available for testing. */
private final GcsService gcsService = GcsServiceFactory.createGcsService(); private final GcsService gcsService = GcsServiceFactory.createGcsService();

View file

@ -40,7 +40,6 @@ import com.google.common.collect.Lists;
import com.google.common.primitives.Longs; import com.google.common.primitives.Longs;
import com.googlecode.objectify.Key; import com.googlecode.objectify.Key;
import com.googlecode.objectify.ObjectifyService; import com.googlecode.objectify.ObjectifyService;
import google.registry.config.TestRegistryConfig;
import google.registry.model.ImmutableObject; import google.registry.model.ImmutableObject;
import google.registry.model.ofy.CommitLogBucket; import google.registry.model.ofy.CommitLogBucket;
import google.registry.model.ofy.CommitLogCheckpoint; import google.registry.model.ofy.CommitLogCheckpoint;
@ -96,11 +95,6 @@ public class RestoreCommitLogsActionTest {
action.diffLister.gcsService = gcsService; action.diffLister.gcsService = gcsService;
action.diffLister.gcsBucket = GCS_BUCKET; action.diffLister.gcsBucket = GCS_BUCKET;
action.diffLister.executor = newDirectExecutorService(); action.diffLister.executor = newDirectExecutorService();
configRule.override(new TestRegistryConfig() {
@Override
public int getCommitLogBucketCount() {
return 3;
}});
} }
@Test @Test

View file

@ -53,7 +53,7 @@ public class ContactResourceTest extends EntityTestCase {
public void setUp() throws Exception { public void setUp() throws Exception {
createTld("foobar"); createTld("foobar");
// Set up a new persisted ContactResource entity. // Set up a new persisted ContactResource entity.
contactResource = cloneAndSetAutoTimestamps( contactResource = persistResource(cloneAndSetAutoTimestamps(
new ContactResource.Builder() new ContactResource.Builder()
.setContactId("contact_id") .setContactId("contact_id")
.setRepoId("1-FOOBAR") .setRepoId("1-FOOBAR")
@ -113,8 +113,7 @@ public class ContactResourceTest extends EntityTestCase {
.setTransferStatus(TransferStatus.SERVER_APPROVED) .setTransferStatus(TransferStatus.SERVER_APPROVED)
.setTransferRequestTrid(Trid.create("client trid")) .setTransferRequestTrid(Trid.create("client trid"))
.build()) .build())
.build()); .build()));
persistResource(contactResource);
} }
@Test @Test

View file

@ -65,7 +65,7 @@ public class DomainApplicationTest extends EntityTestCase {
public void setUp() throws Exception { public void setUp() throws Exception {
createTld("com"); createTld("com");
// Set up a new persisted domain application entity. // Set up a new persisted domain application entity.
domainApplication = cloneAndSetAutoTimestamps( domainApplication = persistResource(cloneAndSetAutoTimestamps(
new DomainApplication.Builder() new DomainApplication.Builder()
.setFullyQualifiedDomainName("example.com") .setFullyQualifiedDomainName("example.com")
.setRepoId("1-COM") .setRepoId("1-COM")
@ -96,8 +96,7 @@ public class DomainApplicationTest extends EntityTestCase {
.setEncodedSignedMarks(ImmutableList.of(EncodedSignedMark.create("base64", "abcdefg="))) .setEncodedSignedMarks(ImmutableList.of(EncodedSignedMark.create("base64", "abcdefg=")))
.setApplicationStatus(ApplicationStatus.ALLOCATED) .setApplicationStatus(ApplicationStatus.ALLOCATED)
.setAuctionPrice(Money.of(USD, 11)) .setAuctionPrice(Money.of(USD, 11))
.build()); .build()));
persistResource(domainApplication);
} }
@Test @Test

View file

@ -107,7 +107,7 @@ public class DomainResourceTest extends EntityTestCase {
Key<PollMessage.OneTime> onetimePollKey = Key<PollMessage.OneTime> onetimePollKey =
Key.create(historyEntryKey, PollMessage.OneTime.class, 1); Key.create(historyEntryKey, PollMessage.OneTime.class, 1);
// Set up a new persisted domain entity. // Set up a new persisted domain entity.
domain = cloneAndSetAutoTimestamps( domain = persistResource(cloneAndSetAutoTimestamps(
new DomainResource.Builder() new DomainResource.Builder()
.setFullyQualifiedDomainName("example.com") .setFullyQualifiedDomainName("example.com")
.setRepoId("4-COM") .setRepoId("4-COM")
@ -158,8 +158,7 @@ public class DomainResourceTest extends EntityTestCase {
.setApplication(Key.create(DomainApplication.class, 1)) .setApplication(Key.create(DomainApplication.class, 1))
.addGracePeriod(GracePeriod.create( .addGracePeriod(GracePeriod.create(
GracePeriodStatus.ADD, clock.nowUtc().plusDays(1), "registrar", null)) GracePeriodStatus.ADD, clock.nowUtc().plusDays(1), "registrar", null))
.build()); .build()));
persistResource(domain);
} }
@Test @Test

View file

@ -72,21 +72,21 @@ public class HostResourceTest extends EntityTestCase {
.build()) .build())
.build()); .build());
hostResource = hostResource =
cloneAndSetAutoTimestamps( persistResource(
new HostResource.Builder() cloneAndSetAutoTimestamps(
.setRepoId("DEADBEEF-COM") new HostResource.Builder()
.setFullyQualifiedHostName("ns1.example.com") .setRepoId("DEADBEEF-COM")
.setCreationClientId("a registrar") .setFullyQualifiedHostName("ns1.example.com")
.setLastEppUpdateTime(clock.nowUtc()) .setCreationClientId("a registrar")
.setLastEppUpdateClientId("another registrar") .setLastEppUpdateTime(clock.nowUtc())
.setLastTransferTime(clock.nowUtc()) .setLastEppUpdateClientId("another registrar")
.setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1"))) .setLastTransferTime(clock.nowUtc())
.setStatusValues(ImmutableSet.of(StatusValue.OK)) .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
.setSuperordinateDomain( .setStatusValues(ImmutableSet.of(StatusValue.OK))
Key.create( .setSuperordinateDomain(
loadByForeignKey(DomainResource.class, "example.com", clock.nowUtc()))) Key.create(
.build()); loadByForeignKey(DomainResource.class, "example.com", clock.nowUtc())))
persistResource(hostResource); .build()));
} }
@Test @Test

View file

@ -24,11 +24,9 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
import com.google.common.base.Suppliers; import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.googlecode.objectify.annotation.Cache; import com.googlecode.objectify.annotation.Cache;
import google.registry.config.TestRegistryConfig;
import google.registry.testing.AppEngineRule; import google.registry.testing.AppEngineRule;
import google.registry.testing.ExceptionRule; import google.registry.testing.ExceptionRule;
import google.registry.testing.InjectRule; import google.registry.testing.InjectRule;
import google.registry.testing.RegistryConfigRule;
import org.junit.Before; import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
@ -44,9 +42,6 @@ public class CommitLogBucketTest {
.withDatastore() .withDatastore()
.build(); .build();
@Rule
public final RegistryConfigRule configRule = new RegistryConfigRule();
@Rule @Rule
public final InjectRule inject = new InjectRule(); public final InjectRule inject = new InjectRule();
@ -57,12 +52,6 @@ public class CommitLogBucketTest {
@Before @Before
public void before() { public void before() {
// Use 10 buckets to make the tests below more realistic.
configRule.override(new TestRegistryConfig() {
@Override
public int getCommitLogBucketCount() {
return 10;
}});
// Save the bucket with some non-default properties set so that we can distinguish a correct // Save the bucket with some non-default properties set so that we can distinguish a correct
// load from one that returns a newly created bucket instance. // load from one that returns a newly created bucket instance.
bucket = persistResource( bucket = persistResource(
@ -108,19 +97,17 @@ public class CommitLogBucketTest {
@Test @Test
public void test_loadBucket_forNonexistentBucket_returnsNewBucket() { public void test_loadBucket_forNonexistentBucket_returnsNewBucket() {
assertThat(loadBucket(getBucketKey(10))).isEqualTo( assertThat(loadBucket(getBucketKey(3))).isEqualTo(
new CommitLogBucket.Builder().setBucketNum(10).build()); new CommitLogBucket.Builder().setBucketNum(3).build());
} }
@Test @Test
public void test_loadAllBuckets_loadsExistingBuckets_orNewOnesIfNonexistent() { public void test_loadAllBuckets_loadsExistingBuckets_orNewOnesIfNonexistent() {
ImmutableSet<CommitLogBucket> buckets = loadAllBuckets(); ImmutableSet<CommitLogBucket> buckets = loadAllBuckets();
assertThat(buckets).hasSize(10); assertThat(buckets).hasSize(3);
assertThat(buckets).contains(bucket); assertThat(buckets).contains(bucket);
for (int i = 2; i <= 10; ++i) { assertThat(buckets).contains(new CommitLogBucket.Builder().setBucketNum(2).build());
assertThat(buckets).contains( assertThat(buckets).contains(new CommitLogBucket.Builder().setBucketNum(3).build());
new CommitLogBucket.Builder().setBucketNum(i).build());
}
} }
@Test @Test

View file

@ -19,12 +19,10 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.joda.time.DateTimeZone.UTC; import static org.joda.time.DateTimeZone.UTC;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import google.registry.config.TestRegistryConfig;
import google.registry.testing.AppEngineRule; import google.registry.testing.AppEngineRule;
import google.registry.testing.ExceptionRule; import google.registry.testing.ExceptionRule;
import google.registry.testing.RegistryConfigRule; import google.registry.testing.RegistryConfigRule;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
@ -49,16 +47,6 @@ public class CommitLogCheckpointTest {
private static final DateTime T2 = START_OF_TIME.plusMillis(1); private static final DateTime T2 = START_OF_TIME.plusMillis(1);
private static final DateTime T3 = START_OF_TIME.plusMillis(2); private static final DateTime T3 = START_OF_TIME.plusMillis(2);
@Before
public void before() {
// Use 3 buckets to make the tests below more realistic.
configRule.override(new TestRegistryConfig() {
@Override
public int getCommitLogBucketCount() {
return 3;
}});
}
@Test @Test
public void test_getCheckpointTime() { public void test_getCheckpointTime() {
DateTime now = DateTime.now(UTC); DateTime now = DateTime.now(UTC);
@ -74,18 +62,6 @@ public class CommitLogCheckpointTest {
assertThat(checkpoint.getBucketTimestamps()).containsExactly(1, T1, 2, T2, 3, T3); assertThat(checkpoint.getBucketTimestamps()).containsExactly(1, T1, 2, T2, 3, T3);
} }
@Test
public void test_getBucketTimestamps_whenOnlyOneBucket_stillWorks() {
configRule.override(new TestRegistryConfig() {
@Override
public int getCommitLogBucketCount() {
return 1;
}});
CommitLogCheckpoint checkpoint =
CommitLogCheckpoint.create(DateTime.now(UTC), ImmutableMap.of(1, T1));
assertThat(checkpoint.getBucketTimestamps()).containsExactly(1, T1);
}
@Test @Test
public void test_create_notEnoughBucketTimestamps_throws() { public void test_create_notEnoughBucketTimestamps_throws() {
thrown.expect(IllegalArgumentException.class, "Bucket ids are incorrect"); thrown.expect(IllegalArgumentException.class, "Bucket ids are incorrect");

View file

@ -40,6 +40,7 @@ import google.registry.model.ofy.CommitLogMutation;
import google.registry.testing.FakeResponse; import google.registry.testing.FakeResponse;
import google.registry.testing.mapreduce.MapreduceTestCase; import google.registry.testing.mapreduce.MapreduceTestCase;
import java.util.List; import java.util.List;
import org.joda.time.DateTime;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.junit.runners.JUnit4; import org.junit.runners.JUnit4;
@ -72,10 +73,11 @@ public class KillAllCommitLogsActionTest extends MapreduceTestCase<KillAllCommit
newContactResource(String.format("abc%d", nextContactId++))); newContactResource(String.format("abc%d", nextContactId++)));
} }
persistResource(CommitLogCheckpointRoot.create(START_OF_TIME.plusDays(1))); persistResource(CommitLogCheckpointRoot.create(START_OF_TIME.plusDays(1)));
DateTime bucketTime = START_OF_TIME.plusDays(2);
persistResource( persistResource(
CommitLogCheckpoint.create( CommitLogCheckpoint.create(
START_OF_TIME.plusDays(1), START_OF_TIME.plusDays(1),
ImmutableMap.of(1, START_OF_TIME.plusDays(2)))); ImmutableMap.of(1, bucketTime, 2, bucketTime, 3, bucketTime)));
for (Class<?> clazz : AFFECTED_TYPES) { for (Class<?> clazz : AFFECTED_TYPES) {
assertThat(ofy().load().type(clazz)).named("entities of type " + clazz).isNotEmpty(); assertThat(ofy().load().type(clazz)).named("entities of type " + clazz).isNotEmpty();
} }