diff --git a/core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java b/core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java index aeb1e9092..9dbe1f9fd 100644 --- a/core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java +++ b/core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java @@ -15,6 +15,7 @@ package google.registry.model.registry.label; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; @@ -30,6 +31,7 @@ import com.google.common.collect.Multiset; import com.google.common.util.concurrent.UncheckedExecutionException; import com.googlecode.objectify.Key; import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Ignore; import com.googlecode.objectify.annotation.Parent; import google.registry.model.Buildable; import google.registry.model.ImmutableObject; @@ -42,6 +44,11 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.MappedSuperclass; +import javax.persistence.Transient; import org.joda.time.DateTime; /** @@ -49,25 +56,46 @@ import org.joda.time.DateTime; * * @param The type of the root value being listed, e.g. {@link ReservationType}. * @param The type of domain label entry being listed, e.g. {@link ReservedListEntry} (note, - * must subclass {@link DomainLabelEntry}. + * must subclass {@link DomainLabelEntry}. */ +@MappedSuperclass public abstract class BaseDomainLabelList, R extends DomainLabelEntry> extends ImmutableObject implements Buildable { + @Ignore + @javax.persistence.Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + Long revisionId; + @Id + @Column(nullable = false) String name; - @Parent - Key parent = getCrossTldKey(); + @Parent @Transient Key parent = getCrossTldKey(); - DateTime creationTime; + @Transient DateTime creationTime; + // The list in Cloud SQL is immutable, we only have a creation_timestamp field and it should be + // set to the timestamp when the list is created. In Datastore, we have two fields and the + // lastUpdateTime is set to the current timestamp when creating and updating a list. So, we use + // lastUpdateTime as the creation_timestamp column during the dual-write phase for compatibility. + @Column(name = "creation_timestamp", nullable = false) DateTime lastUpdateTime; + /** Returns the ID of this revision, or throws if null. */ + public long getRevisionId() { + checkState( + revisionId != null, + "revisionId is null because this object has not been persisted to the database yet"); + return revisionId; + } + + /** Returns the name of the reserved list. */ public String getName() { return name; } + /** Returns the creation time of this revision of the reserved list. */ public DateTime getCreationTime() { return creationTime; } @@ -183,6 +211,9 @@ public abstract class BaseDomainLabelList, R extends Dom @Override public T build() { checkArgument(!isNullOrEmpty(getInstance().name), "List must have a name"); + // The list is immutable in Cloud SQL, so make sure the revision id is not set when the + // builder object is created from a list object + getInstance().revisionId = null; return super.build(); } } diff --git a/core/src/main/java/google/registry/model/registry/label/DomainLabelEntry.java b/core/src/main/java/google/registry/model/registry/label/DomainLabelEntry.java index fe46a2685..435c6b5b8 100644 --- a/core/src/main/java/google/registry/model/registry/label/DomainLabelEntry.java +++ b/core/src/main/java/google/registry/model/registry/label/DomainLabelEntry.java @@ -23,16 +23,20 @@ import com.google.common.net.InternetDomainName; import com.googlecode.objectify.annotation.Id; import google.registry.model.Buildable.GenericBuilder; import google.registry.model.ImmutableObject; +import javax.persistence.Column; +import javax.persistence.MappedSuperclass; /** * Represents a label entry parsed from a line in a reserved/premium list txt file. * * @param The type of the value stored for the domain label, e.g. {@link ReservationType}. */ +@MappedSuperclass public abstract class DomainLabelEntry, D extends DomainLabelEntry> extends ImmutableObject implements Comparable { @Id + @Column(name = "domain_label", insertable = false, updatable = false) String label; String comment; diff --git a/core/src/main/java/google/registry/model/registry/label/ReservedList.java b/core/src/main/java/google/registry/model/registry/label/ReservedList.java index 0ec0583ea..1f7c5b08d 100644 --- a/core/src/main/java/google/registry/model/registry/label/ReservedList.java +++ b/core/src/main/java/google/registry/model/registry/label/ReservedList.java @@ -16,11 +16,8 @@ package google.registry.model.registry.label; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration; -import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; -import static google.registry.model.ofy.ObjectifyService.ofy; import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED; import static google.registry.util.CollectionUtils.nullToEmpty; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -30,12 +27,8 @@ import com.google.common.base.Splitter; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.MapDifference; -import com.google.common.collect.Maps; -import com.google.common.flogger.FluentLogger; import com.google.common.util.concurrent.UncheckedExecutionException; import com.googlecode.objectify.Key; import com.googlecode.objectify.annotation.Embed; @@ -45,45 +38,58 @@ import com.googlecode.objectify.mapper.Mapper; import google.registry.model.Buildable; import google.registry.model.registry.Registry; import google.registry.model.registry.label.DomainLabelMetrics.MetricsReservedListMatch; -import google.registry.schema.replay.DatastoreEntity; -import google.registry.schema.replay.SqlEntity; -import google.registry.schema.tld.ReservedList.ReservedEntry; -import google.registry.schema.tld.ReservedListDao; +import google.registry.schema.replay.DatastoreAndSqlEntity; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; +import javax.persistence.CollectionTable; +import javax.persistence.Column; +import javax.persistence.ElementCollection; +import javax.persistence.Embeddable; +import javax.persistence.Index; +import javax.persistence.JoinColumn; +import javax.persistence.MapKeyColumn; +import javax.persistence.Table; import org.joda.time.DateTime; /** - * A reserved list entity, persisted to Datastore, that is used to check domain label reservations. + * A list of reserved domain labels that are blocked from being registered for various reasons. + * + *

Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by + * the database. So, if a retry of insertion happens after the previous attempt unexpectedly + * succeeds, we will end up with having two exact same reserved lists that differ only by + * revisionId. This is fine though, because we only use the list with the highest revisionId. */ @Entity +@javax.persistence.Entity +@Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")}) public final class ReservedList extends BaseDomainLabelList - implements DatastoreEntity { - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + implements DatastoreAndSqlEntity { @Mapify(ReservedListEntry.LabelMapper.class) + @ElementCollection + @CollectionTable( + name = "ReservedEntry", + joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId")) + @MapKeyColumn(name = "domain_label") Map reservedListMap; + @Column(nullable = false) boolean shouldPublish = true; - @Override - public ImmutableList toSqlEntities() { - return ImmutableList.of(); // ReservedList is dual-written - } - /** * A reserved list entry entity, persisted to Datastore, that represents a single label and its * reservation type. */ @Embed - public static class ReservedListEntry - extends DomainLabelEntry implements Buildable { + @Embeddable + public static class ReservedListEntry extends DomainLabelEntry + implements Buildable { + @Column(nullable = false) ReservationType reservationType; /** Mapper for use with @Mapify */ @@ -150,6 +156,7 @@ public final class ReservedList return shouldPublish; } + /** Returns a {@link Map} of domain labels to {@link ReservedListEntry}. */ public ImmutableMap getReservedListEntries() { return ImmutableMap.copyOf(nullToEmpty(reservedListMap)); } @@ -239,65 +246,10 @@ public final class ReservedList new CacheLoader() { @Override public ReservedList load(String listName) { - ReservedList datastoreList = - ofy() - .load() - .type(ReservedList.class) - .parent(getCrossTldKey()) - .id(listName) - .now(); - // Also load the list from Cloud SQL, compare the two lists, and log if different. - try { - loadAndCompareCloudSqlList(datastoreList); - } catch (Throwable t) { - logger.atSevere().withCause(t).log("Error comparing reserved lists."); - } - return datastoreList; + return ReservedListDualWriteDao.getLatestRevision(listName).orElse(null); } }); - private static final void loadAndCompareCloudSqlList(ReservedList datastoreList) { - Optional maybeCloudSqlList = - ReservedListDao.getLatestRevision(datastoreList.getName()); - if (maybeCloudSqlList.isPresent()) { - Map datastoreLabelsToReservations = - datastoreList.reservedListMap.entrySet().parallelStream() - .collect( - toImmutableMap( - Map.Entry::getKey, - entry -> - ReservedEntry.create( - entry.getValue().reservationType, entry.getValue().comment))); - - google.registry.schema.tld.ReservedList cloudSqlList = maybeCloudSqlList.get(); - MapDifference diff = - Maps.difference(datastoreLabelsToReservations, cloudSqlList.getLabelsToReservations()); - if (!diff.areEqual()) { - if (diff.entriesDiffering().size() > 10) { - logger.atWarning().log( - String.format( - "Unequal reserved lists detected, Cloud SQL list with revision" - + " id %d has %d different records than the current" - + " Datastore list.", - cloudSqlList.getRevisionId(), diff.entriesDiffering().size())); - } else { - StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n"); - diff.entriesDiffering() - .forEach( - (label, valueDiff) -> - diffMessage.append( - String.format( - "Domain label %s has entry %s in Datastore and entry" - + " %s in Cloud SQL.\n", - label, valueDiff.leftValue(), valueDiff.rightValue()))); - logger.atWarning().log(diffMessage.toString()); - } - } - } else { - logger.atWarning().log("Reserved list in Cloud SQL is empty."); - } - } - /** * Gets the {@link ReservationType} of a label in a single ReservedList, or returns an absent * Optional if none exists in the list. diff --git a/core/src/main/java/google/registry/model/registry/label/ReservedListDualWriteDao.java b/core/src/main/java/google/registry/model/registry/label/ReservedListDualWriteDao.java new file mode 100644 index 000000000..5ce62c83e --- /dev/null +++ b/core/src/main/java/google/registry/model/registry/label/ReservedListDualWriteDao.java @@ -0,0 +1,124 @@ +// Copyright 2020 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model.registry.label; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static google.registry.model.common.EntityGroupRoot.getCrossTldKey; +import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm; + +import com.google.common.collect.MapDifference; +import com.google.common.collect.MapDifference.ValueDifference; +import com.google.common.collect.Maps; +import com.google.common.flogger.FluentLogger; +import com.googlecode.objectify.Key; +import google.registry.model.registry.label.ReservedList.ReservedListEntry; +import google.registry.persistence.VKey; +import java.util.Map; +import java.util.Optional; + +/** + * A {@link ReservedList} DAO that does dual-write and dual-read against Datastore and Cloud SQL. It + * still uses Datastore as the primary storage and suppresses any exception thrown by Cloud SQL. + * + *

TODO(b/160993806): Delete this DAO and switch to use the SQL only DAO after migrating to Cloud + * SQL. + */ +public class ReservedListDualWriteDao { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private ReservedListDualWriteDao() {} + + /** Persist a new reserved list to Cloud SQL. */ + public static void save(ReservedList reservedList) { + ofyTm().transact(() -> ofyTm().saveNewOrUpdate(reservedList)); + try { + logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName()); + ReservedListSqlDao.save(reservedList); + logger.atInfo().log( + "Saved reserved list %s with %d entries to Cloud SQL", + reservedList.getName(), reservedList.getReservedListEntries().size()); + } catch (Throwable t) { + logger.atSevere().withCause(t).log("Error saving the reserved list to Cloud SQL."); + } + } + + /** + * Returns the most recent revision of the {@link ReservedList} with the specified name, if it + * exists. + */ + public static Optional getLatestRevision(String reservedListName) { + Optional maybeDatastoreList = + ofyTm() + .maybeLoad( + VKey.createOfy( + ReservedList.class, + Key.create(getCrossTldKey(), ReservedList.class, reservedListName))); + try { + // Also load the list from Cloud SQL, compare the two lists, and log if different. + maybeDatastoreList.ifPresent(ReservedListDualWriteDao::loadAndCompareCloudSqlList); + } catch (Throwable t) { + logger.atSevere().withCause(t).log("Error comparing reserved lists."); + } + return maybeDatastoreList; + } + + private static void loadAndCompareCloudSqlList(ReservedList datastoreList) { + Optional maybeCloudSqlList = + ReservedListSqlDao.getLatestRevision(datastoreList.getName()); + if (maybeCloudSqlList.isPresent()) { + Map datastoreLabelsToReservations = + datastoreList.reservedListMap.entrySet().parallelStream() + .collect( + toImmutableMap( + Map.Entry::getKey, + entry -> + ReservedListEntry.create( + entry.getKey(), + entry.getValue().reservationType, + entry.getValue().comment))); + + ReservedList cloudSqlList = maybeCloudSqlList.get(); + MapDifference diff = + Maps.difference(datastoreLabelsToReservations, cloudSqlList.reservedListMap); + if (!diff.areEqual()) { + if (diff.entriesDiffering().size() > 10) { + logger.atWarning().log( + String.format( + "Unequal reserved lists detected, Cloud SQL list with revision" + + " id %d has %d different records than the current" + + " Datastore list.", + cloudSqlList.getRevisionId(), diff.entriesDiffering().size())); + } else { + StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n"); + diff.entriesDiffering().entrySet().stream() + .forEach( + entry -> { + String label = entry.getKey(); + ValueDifference valueDiff = entry.getValue(); + diffMessage.append( + String.format( + "Domain label %s has entry %s in Datastore and entry" + + " %s in Cloud SQL.\n", + label, valueDiff.leftValue(), valueDiff.rightValue())); + }); + logger.atWarning().log(diffMessage.toString()); + } + } + } else { + logger.atWarning().log("Reserved list in Cloud SQL is empty."); + } + } +} diff --git a/core/src/main/java/google/registry/schema/tld/ReservedListDao.java b/core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java similarity index 70% rename from core/src/main/java/google/registry/schema/tld/ReservedListDao.java rename to core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java index 847530b50..26b61e220 100644 --- a/core/src/main/java/google/registry/schema/tld/ReservedListDao.java +++ b/core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java @@ -12,20 +12,46 @@ // See the License for the specific language governing permissions and // limitations under the License. -package google.registry.schema.tld; +package google.registry.model.registry.label; import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; +import static google.registry.util.PreconditionsUtils.checkArgumentNotNull; -import com.google.common.util.concurrent.UncheckedExecutionException; import java.util.Optional; -import java.util.concurrent.ExecutionException; -/** Data access object class for {@link ReservedList} */ -public class ReservedListDao { +/** + * A {@link ReservedList} DAO for Cloud SQL. + * + *

TODO(b/160993806): Rename this class to ReservedListDao after migrating to Cloud SQL. + */ +public class ReservedListSqlDao { + + private ReservedListSqlDao() {} /** Persist a new reserved list to Cloud SQL. */ public static void save(ReservedList reservedList) { - jpaTm().transact(() -> jpaTm().getEntityManager().persist(reservedList)); + checkArgumentNotNull(reservedList, "Must specify reservedList"); + jpaTm().transact(() -> jpaTm().saveNew(reservedList)); + } + + /** + * Returns the most recent revision of the {@link ReservedList} with the specified name, if it + * exists. + */ + public static Optional getLatestRevision(String reservedListName) { + return jpaTm() + .transact( + () -> + jpaTm() + .getEntityManager() + .createQuery( + "FROM ReservedList rl LEFT JOIN FETCH rl.reservedListMap WHERE" + + " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl" + + " WHERE subrl.name = :name)", + ReservedList.class) + .setParameter("name", reservedListName) + .getResultStream() + .findFirst()); } /** @@ -46,39 +72,4 @@ public class ReservedListDao { .size() > 0); } - - /** - * Returns the most recent revision of the {@link ReservedList} with the specified name, if it - * exists. TODO(shicong): Change this method to package level access after dual-read phase. - */ - public static Optional getLatestRevision(String reservedListName) { - return jpaTm() - .transact( - () -> - jpaTm() - .getEntityManager() - .createQuery( - "FROM ReservedList rl LEFT JOIN FETCH rl.labelsToReservations WHERE" - + " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl" - + " WHERE subrl.name = :name)", - ReservedList.class) - .setParameter("name", reservedListName) - .getResultStream() - .findFirst()); - } - - /** - * Returns the most recent revision of the {@link ReservedList} with the specified name, from - * cache. - */ - public static Optional getLatestRevisionCached(String reservedListName) { - try { - return ReservedListCache.cacheReservedLists.get(reservedListName); - } catch (ExecutionException e) { - throw new UncheckedExecutionException( - "Could not retrieve reserved list named " + reservedListName, e); - } - } - - private ReservedListDao() {} } diff --git a/core/src/main/java/google/registry/persistence/VKey.java b/core/src/main/java/google/registry/persistence/VKey.java index b464035ac..a390e7c30 100644 --- a/core/src/main/java/google/registry/persistence/VKey.java +++ b/core/src/main/java/google/registry/persistence/VKey.java @@ -58,6 +58,31 @@ public class VKey extends ImmutableObject implements Serializable { return new VKey(kind, null, sqlKey); } + /** Creates a {@link VKey} which only contains the ofy primary key. */ + public static VKey createOfy( + Class kind, com.googlecode.objectify.Key ofyKey) { + checkArgumentNotNull(kind, "kind must not be null"); + checkArgumentNotNull(ofyKey, "ofyKey must not be null"); + return new VKey(kind, ofyKey, null); + } + + /** + * Creates a {@link VKey} which only contains the ofy primary key by specifying the id of the + * {@link Key}. + */ + public static VKey createOfy(Class kind, long id) { + return createOfy(kind, Key.create(kind, id)); + } + + /** + * Creates a {@link VKey} which only contains the ofy primary key by specifying the name of the + * {@link Key}. + */ + public static VKey createOfy(Class kind, String name) { + checkArgumentNotNull(kind, "name must not be null"); + return createOfy(kind, Key.create(kind, name)); + } + /** Creates a {@link VKey} which only contains both sql and ofy primary key. */ public static VKey create( Class kind, Object sqlKey, com.googlecode.objectify.Key ofyKey) { diff --git a/core/src/main/java/google/registry/schema/tld/ReservedList.java b/core/src/main/java/google/registry/schema/tld/ReservedList.java deleted file mode 100644 index ce77660ae..000000000 --- a/core/src/main/java/google/registry/schema/tld/ReservedList.java +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.schema.tld; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableList.sortedCopyOf; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static google.registry.util.DomainNameUtils.canonicalizeDomainName; - -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import google.registry.model.CreateAutoTimestamp; -import google.registry.model.ImmutableObject; -import google.registry.model.registry.label.ReservationType; -import google.registry.schema.replay.DatastoreEntity; -import google.registry.schema.replay.SqlEntity; -import java.util.Map; -import java.util.stream.Stream; -import javax.annotation.Nullable; -import javax.persistence.CollectionTable; -import javax.persistence.Column; -import javax.persistence.ElementCollection; -import javax.persistence.Embeddable; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Index; -import javax.persistence.JoinColumn; -import javax.persistence.MapKeyColumn; -import javax.persistence.Table; -import org.joda.time.DateTime; - -/** - * A list of reserved domain labels that are blocked from being registered for various reasons. - * - *

Note that the primary key of this entity is {@link #revisionId}, which is auto-generated by - * the database. So, if a retry of insertion happens after the previous attempt unexpectedly - * succeeds, we will end up with having two exact same reserved lists that differ only by - * revisionId. This is fine though, because we only use the list with the highest revisionId. - */ -@Entity -@Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")}) -public class ReservedList extends ImmutableObject implements SqlEntity { - - @Column(nullable = false) - private String name; - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(nullable = false) - private Long revisionId; - - @Column(nullable = false) - private CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null); - - @Column(nullable = false) - private Boolean shouldPublish; - - @ElementCollection - @CollectionTable( - name = "ReservedEntry", - joinColumns = @JoinColumn(name = "revisionId", referencedColumnName = "revisionId")) - @MapKeyColumn(name = "domainLabel") - private Map labelsToReservations; - - @Override - public ImmutableList toDatastoreEntities() { - return ImmutableList.of(); // ReservedList is dual-written\ - } - - @Embeddable - public static class ReservedEntry extends ImmutableObject { - @Column(nullable = false) - private ReservationType reservationType; - - @Column(nullable = true) - private String comment; - - private ReservedEntry(ReservationType reservationType, @Nullable String comment) { - this.reservationType = reservationType; - this.comment = comment; - } - - // Hibernate requires this default constructor. - private ReservedEntry() {} - - /** Constructs a {@link ReservedEntry} object. */ - public static ReservedEntry create(ReservationType reservationType, @Nullable String comment) { - return new ReservedEntry(reservationType, comment); - } - - /** Returns the reservation type for this entry. */ - public ReservationType getReservationType() { - return reservationType; - } - - /** Returns the comment for this entry. Retruns null if there is no comment. */ - public String getComment() { - return comment; - } - } - - private ReservedList( - String name, Boolean shouldPublish, Map labelsToReservations) { - this.name = name; - this.shouldPublish = shouldPublish; - this.labelsToReservations = labelsToReservations; - } - - // Hibernate requires this default constructor. - private ReservedList() {} - - /** Constructs a {@link ReservedList} object. */ - public static ReservedList create( - String name, Boolean shouldPublish, Map labelsToReservations) { - ImmutableList invalidLabels = - labelsToReservations.entrySet().parallelStream() - .flatMap( - entry -> { - String label = entry.getKey(); - if (label.equals(canonicalizeDomainName(label))) { - return Stream.empty(); - } else { - return Stream.of(label); - } - }) - .collect(toImmutableList()); - checkArgument( - invalidLabels.isEmpty(), - "Label(s) [%s] must be in puny-coded, lower-case form", - Joiner.on(",").join(sortedCopyOf(invalidLabels))); - return new ReservedList(name, shouldPublish, labelsToReservations); - } - - /** Returns the name of the reserved list. */ - public String getName() { - return name; - } - - /** Returns the ID of this revision, or throws if null. */ - public Long getRevisionId() { - checkState( - revisionId != null, - "revisionId is null because this object has not been persisted to the database yet"); - return revisionId; - } - - /** Returns the creation time of this revision of the reserved list. */ - public DateTime getCreationTimestamp() { - return creationTimestamp.getTimestamp(); - } - - /** Returns a {@link Map} of domain labels to {@link ReservedEntry}. */ - public ImmutableMap getLabelsToReservations() { - return ImmutableMap.copyOf(labelsToReservations); - } - - /** Returns true if the reserved list should be published. */ - public Boolean getShouldPublish() { - return shouldPublish; - } -} diff --git a/core/src/main/java/google/registry/schema/tld/ReservedListCache.java b/core/src/main/java/google/registry/schema/tld/ReservedListCache.java deleted file mode 100644 index 97ac9e88d..000000000 --- a/core/src/main/java/google/registry/schema/tld/ReservedListCache.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.schema.tld; - -import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import google.registry.util.NonFinalForTesting; -import java.util.Optional; -import org.joda.time.Duration; - -/** Caching utils for {@link ReservedList} */ -public class ReservedListCache { - - /** - * In-memory cache for reserved lists. - * - *

This is cached for a shorter duration because we need to periodically reload from the DB to - * check if a new revision has been published, and if so, then use that. - */ - @NonFinalForTesting - static LoadingCache> cacheReservedLists = - createCacheReservedLists(getDomainLabelListCacheDuration()); - - @VisibleForTesting - static LoadingCache> createCacheReservedLists( - Duration cachePersistDuration) { - return CacheBuilder.newBuilder() - .expireAfterWrite(cachePersistDuration.getMillis(), MILLISECONDS) - .build( - new CacheLoader>() { - @Override - public Optional load(String reservedListName) { - return ReservedListDao.getLatestRevision(reservedListName); - } - }); - } -} diff --git a/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java b/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java index 3bcd5736d..b77a87e94 100644 --- a/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java +++ b/core/src/main/java/google/registry/tools/CreateOrUpdateReservedListCommand.java @@ -14,23 +14,12 @@ package google.registry.tools; -import static com.google.common.base.Preconditions.checkArgument; -import static google.registry.model.registry.label.BaseDomainLabelList.splitOnComment; -import static google.registry.util.DomainNameUtils.canonicalizeDomainName; - import com.beust.jcommander.Parameter; -import com.google.common.base.Splitter; -import com.google.common.collect.HashMultiset; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.google.common.collect.Multiset; import com.google.common.flogger.FluentLogger; -import google.registry.model.registry.label.ReservationType; -import google.registry.schema.tld.ReservedList.ReservedEntry; +import google.registry.model.registry.label.ReservedList; +import google.registry.model.registry.label.ReservedListDualWriteDao; import google.registry.tools.params.PathParameter; import java.nio.file.Path; -import java.util.List; -import java.util.Map; import javax.annotation.Nullable; /** @@ -62,69 +51,22 @@ public abstract class CreateOrUpdateReservedListCommand extends MutatingCommand arity = 1) Boolean shouldPublish; - google.registry.schema.tld.ReservedList cloudSqlReservedList; - - abstract void saveToCloudSql(); + ReservedList reservedList; @Override - protected String execute() throws Exception { - // Save the list to Datastore and output its response. - String output = super.execute(); - logger.atInfo().log(output); - - String cloudSqlMessage = + protected String execute() { + String message = String.format( "Saved reserved list %s with %d entries", - name, cloudSqlReservedList.getLabelsToReservations().size()); + name, reservedList.getReservedListEntries().size()); try { - logger.atInfo().log("Saving reserved list to Cloud SQL for TLD %s", name); - saveToCloudSql(); - logger.atInfo().log(cloudSqlMessage); + logger.atInfo().log("Saving reserved list for TLD %s", name); + ReservedListDualWriteDao.save(reservedList); + logger.atInfo().log(message); } catch (Throwable e) { - cloudSqlMessage = - "Unexpected error saving reserved list to Cloud SQL from nomulus tool command"; - logger.atSevere().withCause(e).log(cloudSqlMessage); + message = "Unexpected error saving reserved list from nomulus tool command"; + logger.atSevere().withCause(e).log(message); } - return cloudSqlMessage; - } - - /** Turns the list CSV data into a map of labels to {@link ReservedEntry}. */ - static ImmutableMap parseToReservationsByLabels(Iterable lines) { - Map labelsToEntries = Maps.newHashMap(); - Multiset duplicateLabels = HashMultiset.create(); - for (String originalLine : lines) { - List lineAndComment = splitOnComment(originalLine); - if (lineAndComment.isEmpty()) { - continue; - } - String line = lineAndComment.get(0); - String comment = lineAndComment.get(1); - List parts = Splitter.on(',').trimResults().splitToList(line); - checkArgument( - parts.size() == 2 || parts.size() == 3, - "Could not parse line in reserved list: %s", - originalLine); - String label = parts.get(0); - checkArgument( - label.equals(canonicalizeDomainName(label)), - "Label '%s' must be in puny-coded, lower-case form", - label); - ReservationType reservationType = ReservationType.valueOf(parts.get(1)); - ReservedEntry reservedEntry = ReservedEntry.create(reservationType, comment); - // Check if the label was already processed for this list (which is an error), and if so, - // accumulate it so that a list of all duplicates can be thrown. - if (labelsToEntries.containsKey(label)) { - duplicateLabels.add(label, duplicateLabels.contains(label) ? 1 : 2); - } else { - labelsToEntries.put(label, reservedEntry); - } - } - if (!duplicateLabels.isEmpty()) { - throw new IllegalStateException( - String.format( - "Reserved list cannot contain duplicate labels. Dupes (with counts) were: %s", - duplicateLabels)); - } - return ImmutableMap.copyOf(labelsToEntries); + return message; } } diff --git a/core/src/main/java/google/registry/tools/CreateReservedListCommand.java b/core/src/main/java/google/registry/tools/CreateReservedListCommand.java index 267da14f1..93aaf7eb1 100644 --- a/core/src/main/java/google/registry/tools/CreateReservedListCommand.java +++ b/core/src/main/java/google/registry/tools/CreateReservedListCommand.java @@ -16,7 +16,6 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; import static google.registry.model.registry.Registries.assertTldExists; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.util.ListNamingUtils.convertFilePathToName; import static java.nio.charset.StandardCharsets.UTF_8; import static org.joda.time.DateTimeZone.UTC; @@ -27,7 +26,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.base.Strings; import google.registry.model.registry.label.ReservedList; -import google.registry.schema.tld.ReservedListDao; import java.nio.file.Files; import java.util.List; import org.joda.time.DateTime; @@ -50,15 +48,14 @@ final class CreateReservedListCommand extends CreateOrUpdateReservedListCommand protected void init() throws Exception { name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name; checkArgument( - !ReservedList.get(name).isPresent(), - "A reserved list already exists by this name"); + !ReservedList.get(name).isPresent(), "A reserved list already exists by this name"); if (!override) { validateListName(name); } DateTime now = DateTime.now(UTC); List allLines = Files.readAllLines(input, UTF_8); boolean shouldPublish = this.shouldPublish == null || this.shouldPublish; - ReservedList reservedList = + reservedList = new ReservedList.Builder() .setName(name) .setReservedListMapFromLines(allLines) @@ -66,23 +63,6 @@ final class CreateReservedListCommand extends CreateOrUpdateReservedListCommand .setCreationTime(now) .setLastUpdateTime(now) .build(); - stageEntityChange(null, reservedList); - cloudSqlReservedList = - google.registry.schema.tld.ReservedList.create( - name, shouldPublish, parseToReservationsByLabels(allLines)); - } - - @Override - void saveToCloudSql() { - jpaTm() - .transact( - () -> { - checkArgument( - !ReservedListDao.checkExists(cloudSqlReservedList.getName()), - "A reserved list of this name already exists: %s.", - cloudSqlReservedList.getName()); - ReservedListDao.save(cloudSqlReservedList); - }); } private static void validateListName(String name) { diff --git a/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java b/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java index 02da6c747..01e7cd5af 100644 --- a/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java +++ b/core/src/main/java/google/registry/tools/UpdateReservedListCommand.java @@ -15,18 +15,17 @@ package google.registry.tools; import static com.google.common.base.Preconditions.checkArgument; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.util.ListNamingUtils.convertFilePathToName; import static java.nio.charset.StandardCharsets.UTF_8; import com.beust.jcommander.Parameters; import com.google.common.base.Strings; import google.registry.model.registry.label.ReservedList; -import google.registry.schema.tld.ReservedListDao; import google.registry.util.SystemClock; import java.nio.file.Files; import java.util.List; import java.util.Optional; +import org.joda.time.DateTime; /** Command to safely update {@link ReservedList} on Datastore. */ @Parameters(separators = " =", commandDescription = "Update a ReservedList in Datastore.") @@ -35,42 +34,20 @@ final class UpdateReservedListCommand extends CreateOrUpdateReservedListCommand @Override protected void init() throws Exception { name = Strings.isNullOrEmpty(name) ? convertFilePathToName(input) : name; - // TODO(shicong): Read existing entry from Cloud SQL Optional existing = ReservedList.get(name); checkArgument( existing.isPresent(), "Could not update reserved list %s because it doesn't exist.", name); boolean shouldPublish = this.shouldPublish == null ? existing.get().getShouldPublish() : this.shouldPublish; List allLines = Files.readAllLines(input, UTF_8); + DateTime now = new SystemClock().nowUtc(); ReservedList.Builder updated = existing .get() .asBuilder() .setReservedListMapFromLines(allLines) - .setLastUpdateTime(new SystemClock().nowUtc()) + .setLastUpdateTime(now) .setShouldPublish(shouldPublish); - stageEntityChange(existing.get(), updated.build()); - cloudSqlReservedList = - google.registry.schema.tld.ReservedList.create( - name, shouldPublish, parseToReservationsByLabels(allLines)); - } - - @Override - void saveToCloudSql() { - jpaTm() - .transact( - () -> { - // This check is currently disabled because, during the Cloud SQL migration, we need - // to be able to update reserved lists in Datastore while simultaneously creating - // their first revision in Cloud SQL (i.e. if they haven't been migrated over yet). - // TODO(shicong): Re-instate this once all reserved lists are migrated to Cloud SQL, - // and add a unit test to verity that an exception will be thrown if - // the reserved list doesn't exist. - // checkArgument( - // ReservedListDao.checkExists(cloudSqlReservedList.getName()), - // "A reserved list of this name doesn't exist: %s.", - // cloudSqlReservedList.getName()); - ReservedListDao.save(cloudSqlReservedList); - }); + reservedList = updated.build(); } } diff --git a/core/src/main/resources/META-INF/persistence.xml b/core/src/main/resources/META-INF/persistence.xml index 65b97aefd..31d9a59c1 100644 --- a/core/src/main/resources/META-INF/persistence.xml +++ b/core/src/main/resources/META-INF/persistence.xml @@ -36,12 +36,12 @@ google.registry.schema.server.Lock google.registry.schema.tld.PremiumList google.registry.schema.tld.PremiumEntry - google.registry.schema.tld.ReservedList google.registry.model.domain.secdns.DelegationSignerData google.registry.model.domain.GracePeriod google.registry.model.poll.PollMessage google.registry.model.poll.PollMessage$OneTime google.registry.model.poll.PollMessage$Autorenew + google.registry.model.registry.label.ReservedList google.registry.persistence.converter.BillingCostTransitionConverter diff --git a/core/src/test/java/google/registry/beam/initsql/BeamJpaModuleTest.java b/core/src/test/java/google/registry/beam/initsql/BeamJpaModuleTest.java index 57249f5e5..d7bebc3ee 100644 --- a/core/src/test/java/google/registry/beam/initsql/BeamJpaModuleTest.java +++ b/core/src/test/java/google/registry/beam/initsql/BeamJpaModuleTest.java @@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat; import google.registry.persistence.NomulusPostgreSql; import google.registry.persistence.transaction.JpaTransactionManager; +import google.registry.testing.DatastoreEntityExtension; import java.io.File; import java.io.IOException; import java.io.PrintStream; @@ -26,6 +27,7 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; @@ -38,6 +40,9 @@ public class BeamJpaModuleTest { @Container public PostgreSQLContainer database = new PostgreSQLContainer(NomulusPostgreSql.getDockerTag()); + @RegisterExtension + public DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); + @TempDir File tempFolder; private File credentialFile; @@ -51,7 +56,7 @@ public class BeamJpaModuleTest { } @Test - public void getJpaTransactionManager_local() { + void getJpaTransactionManager_local() { JpaTransactionManager jpa = DaggerBeamJpaModule_JpaTransactionManagerComponent.builder() .beamJpaModule(new BeamJpaModule(credentialFile.getAbsolutePath())) diff --git a/core/src/test/java/google/registry/model/registry/label/ReservedListSqlDaoTest.java b/core/src/test/java/google/registry/model/registry/label/ReservedListSqlDaoTest.java new file mode 100644 index 000000000..e62ae02e3 --- /dev/null +++ b/core/src/test/java/google/registry/model/registry/label/ReservedListSqlDaoTest.java @@ -0,0 +1,124 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.model.registry.label; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; + +import com.google.common.collect.ImmutableMap; +import google.registry.model.registry.label.ReservedList.ReservedListEntry; +import google.registry.persistence.transaction.JpaTestRules; +import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension; +import google.registry.testing.DatastoreEntityExtension; +import google.registry.testing.FakeClock; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** Unit tests for {@link ReservedListSqlDao}. */ +public class ReservedListSqlDaoTest { + + private final FakeClock fakeClock = new FakeClock(); + + @RegisterExtension + @Order(value = 1) + DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); + + @RegisterExtension + JpaIntegrationWithCoverageExtension jpa = + new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension(); + + private ImmutableMap test_reservations; + + private ReservedList test_reserved_list; + + @BeforeEach + void setUp() { + test_reservations = + ImmutableMap.of( + "food", + ReservedListEntry.create("food", ReservationType.RESERVED_FOR_SPECIFIC_USE, null), + "music", + ReservedListEntry.create("music", ReservationType.FULLY_BLOCKED, "fully blocked")); + + test_reserved_list = + new ReservedList.Builder() + .setName("testlist") + .setLastUpdateTime(fakeClock.nowUtc()) + .setShouldPublish(false) + .setReservedListMap(test_reservations) + .build(); + } + + @Test + public void save_worksSuccessfully() { + ReservedListSqlDao.save(test_reserved_list); + jpaTm() + .transact( + () -> { + ReservedList persistedList = + jpaTm() + .getEntityManager() + .createQuery("FROM ReservedList WHERE name = :name", ReservedList.class) + .setParameter("name", "testlist") + .getSingleResult(); + assertThat(persistedList.getReservedListEntries()) + .containsExactlyEntriesIn(test_reservations); + assertThat(persistedList.getLastUpdateTime()).isEqualTo(fakeClock.nowUtc()); + }); + } + + @Test + public void checkExists_worksSuccessfully() { + assertThat(ReservedListSqlDao.checkExists("testlist")).isFalse(); + ReservedListSqlDao.save(test_reserved_list); + assertThat(ReservedListSqlDao.checkExists("testlist")).isTrue(); + } + + @Test + public void getLatestRevision_worksSuccessfully() { + assertThat(ReservedListSqlDao.getLatestRevision("testlist").isPresent()).isFalse(); + ReservedListSqlDao.save(test_reserved_list); + ReservedList persistedList = ReservedListSqlDao.getLatestRevision("testlist").get(); + assertThat(persistedList.getRevisionId()).isNotNull(); + assertThat(persistedList.getLastUpdateTime()).isEqualTo(fakeClock.nowUtc()); + assertThat(persistedList.getName()).isEqualTo("testlist"); + assertThat(persistedList.getShouldPublish()).isFalse(); + assertThat(persistedList.getReservedListEntries()).containsExactlyEntriesIn(test_reservations); + } + + @Test + public void getLatestRevision_returnsLatestRevision() { + ReservedListSqlDao.save( + new ReservedList.Builder() + .setName("testlist") + .setLastUpdateTime(fakeClock.nowUtc()) + .setShouldPublish(false) + .setReservedListMap( + ImmutableMap.of( + "old", + ReservedListEntry.create( + "old", ReservationType.RESERVED_FOR_SPECIFIC_USE, null))) + .build()); + ReservedListSqlDao.save(test_reserved_list); + ReservedList persistedList = ReservedListSqlDao.getLatestRevision("testlist").get(); + assertThat(persistedList.getRevisionId()).isNotNull(); + assertThat(persistedList.getLastUpdateTime()).isEqualTo(fakeClock.nowUtc()); + assertThat(persistedList.getName()).isEqualTo("testlist"); + assertThat(persistedList.getShouldPublish()).isFalse(); + assertThat(persistedList.getReservedListEntries()).containsExactlyEntriesIn(test_reservations); + } +} diff --git a/core/src/test/java/google/registry/persistence/PersistenceModuleTest.java b/core/src/test/java/google/registry/persistence/PersistenceModuleTest.java index 93c68a8e7..320397fc2 100644 --- a/core/src/test/java/google/registry/persistence/PersistenceModuleTest.java +++ b/core/src/test/java/google/registry/persistence/PersistenceModuleTest.java @@ -16,11 +16,13 @@ package google.registry.persistence; import static com.google.common.truth.Truth.assertThat; +import google.registry.testing.DatastoreEntityExtension; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -33,6 +35,9 @@ public class PersistenceModuleTest { private final PostgreSQLContainer database = new PostgreSQLContainer(NomulusPostgreSql.getDockerTag()); + @RegisterExtension + public DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); + private EntityManagerFactory emf; @BeforeEach diff --git a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java index d582e8584..b6417e4ea 100644 --- a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java +++ b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java @@ -23,6 +23,7 @@ import google.registry.model.history.ContactHistoryTest; import google.registry.model.history.HostHistoryTest; import google.registry.model.poll.PollMessageTest; import google.registry.model.registry.RegistryLockDaoTest; +import google.registry.model.registry.label.ReservedListSqlDaoTest; import google.registry.model.reporting.Spec11ThreatMatchTest; import google.registry.persistence.transaction.JpaEntityCoverage; import google.registry.schema.cursor.CursorDaoTest; @@ -31,7 +32,6 @@ import google.registry.schema.integration.SqlIntegrationTestSuite.BeforeSuiteTes import google.registry.schema.registrar.RegistrarDaoTest; import google.registry.schema.server.LockDaoTest; import google.registry.schema.tld.PremiumListDaoTest; -import google.registry.schema.tld.ReservedListDaoTest; import google.registry.schema.tmch.ClaimsListDaoTest; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -84,8 +84,8 @@ import org.junit.runner.RunWith; PollMessageTest.class, PremiumListDaoTest.class, RegistrarDaoTest.class, + ReservedListSqlDaoTest.class, RegistryLockDaoTest.class, - ReservedListDaoTest.class, Spec11ThreatMatchTest.class, // AfterSuiteTest must be the last entry. See class javadoc for details. AfterSuiteTest.class diff --git a/core/src/test/java/google/registry/schema/server/LockDaoTest.java b/core/src/test/java/google/registry/schema/server/LockDaoTest.java index dd3717278..4cbfda402 100644 --- a/core/src/test/java/google/registry/schema/server/LockDaoTest.java +++ b/core/src/test/java/google/registry/schema/server/LockDaoTest.java @@ -20,11 +20,13 @@ import static google.registry.testing.LogsSubject.assertAboutLogs; import com.google.common.testing.TestLogHandler; import google.registry.persistence.transaction.JpaTestRules; import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension; +import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; import org.joda.time.Duration; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -35,6 +37,10 @@ public class LockDaoTest { private final TestLogHandler logHandler = new TestLogHandler(); private final Logger loggerToIntercept = Logger.getLogger(LockDao.class.getCanonicalName()); + @RegisterExtension + @Order(value = 1) + public DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); + @RegisterExtension public final JpaIntegrationWithCoverageExtension jpa = new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension(); diff --git a/core/src/test/java/google/registry/schema/tld/ReservedListDaoTest.java b/core/src/test/java/google/registry/schema/tld/ReservedListDaoTest.java deleted file mode 100644 index 5e76df7bd..000000000 --- a/core/src/test/java/google/registry/schema/tld/ReservedListDaoTest.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.schema.tld; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; - -import com.google.common.collect.ImmutableMap; -import google.registry.model.registry.label.ReservationType; -import google.registry.persistence.transaction.JpaTestRules; -import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension; -import google.registry.schema.tld.ReservedList.ReservedEntry; -import google.registry.testing.FakeClock; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -/** Unit tests for {@link ReservedListDao}. */ -public class ReservedListDaoTest { - - private final FakeClock fakeClock = new FakeClock(); - - @RegisterExtension - public final JpaIntegrationWithCoverageExtension jpa = - new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension(); - - private static final ImmutableMap TEST_RESERVATIONS = - ImmutableMap.of( - "food", - ReservedEntry.create(ReservationType.RESERVED_FOR_SPECIFIC_USE, null), - "music", - ReservedEntry.create(ReservationType.FULLY_BLOCKED, "fully blocked")); - - @Test - public void save_worksSuccessfully() { - ReservedList reservedList = ReservedList.create("testname", false, TEST_RESERVATIONS); - ReservedListDao.save(reservedList); - jpaTm() - .transact( - () -> { - ReservedList persistedList = - jpaTm() - .getEntityManager() - .createQuery("FROM ReservedList WHERE name = :name", ReservedList.class) - .setParameter("name", "testname") - .getSingleResult(); - assertThat(persistedList.getLabelsToReservations()) - .containsExactlyEntriesIn(TEST_RESERVATIONS); - assertThat(persistedList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc()); - }); - } - - @Test - public void checkExists_worksSuccessfully() { - assertThat(ReservedListDao.checkExists("testlist")).isFalse(); - ReservedListDao.save(ReservedList.create("testlist", false, TEST_RESERVATIONS)); - assertThat(ReservedListDao.checkExists("testlist")).isTrue(); - } - - @Test - public void getLatestRevision_worksSuccessfully() { - assertThat(ReservedListDao.getLatestRevision("testlist").isPresent()).isFalse(); - ReservedListDao.save(ReservedList.create("testlist", false, TEST_RESERVATIONS)); - ReservedList persistedList = ReservedListDao.getLatestRevision("testlist").get(); - assertThat(persistedList.getRevisionId()).isNotNull(); - assertThat(persistedList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc()); - assertThat(persistedList.getName()).isEqualTo("testlist"); - assertThat(persistedList.getShouldPublish()).isFalse(); - assertThat(persistedList.getLabelsToReservations()).containsExactlyEntriesIn(TEST_RESERVATIONS); - } - - @Test - public void getLatestRevision_returnsLatestRevision() { - ReservedListDao.save( - ReservedList.create( - "testlist", - false, - ImmutableMap.of( - "old", ReservedEntry.create(ReservationType.RESERVED_FOR_SPECIFIC_USE, null)))); - ReservedListDao.save(ReservedList.create("testlist", false, TEST_RESERVATIONS)); - ReservedList persistedList = ReservedListDao.getLatestRevision("testlist").get(); - assertThat(persistedList.getRevisionId()).isNotNull(); - assertThat(persistedList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc()); - assertThat(persistedList.getName()).isEqualTo("testlist"); - assertThat(persistedList.getShouldPublish()).isFalse(); - assertThat(persistedList.getLabelsToReservations()).containsExactlyEntriesIn(TEST_RESERVATIONS); - } - - @Test - public void getLatestRevisionCached_worksSuccessfully() { - ReservedListDao.save(ReservedList.create("testlist", false, TEST_RESERVATIONS)); - ReservedList persistedList = ReservedListDao.getLatestRevisionCached("testlist").get(); - assertThat(persistedList.getRevisionId()).isNotNull(); - assertThat(persistedList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc()); - assertThat(persistedList.getName()).isEqualTo("testlist"); - assertThat(persistedList.getShouldPublish()).isFalse(); - assertThat(persistedList.getLabelsToReservations()).containsExactlyEntriesIn(TEST_RESERVATIONS); - } -} diff --git a/core/src/test/java/google/registry/schema/tld/ReservedListTest.java b/core/src/test/java/google/registry/schema/tld/ReservedListTest.java deleted file mode 100644 index 1e0571775..000000000 --- a/core/src/test/java/google/registry/schema/tld/ReservedListTest.java +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.schema.tld; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableMap; -import google.registry.model.registry.label.ReservationType; -import google.registry.schema.tld.ReservedList.ReservedEntry; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link ReservedList} */ -@RunWith(JUnit4.class) -public class ReservedListTest { - - @Test - public void verifyConstructorAndGetters_workCorrectly() { - ReservedList reservedList = - ReservedList.create( - "app", - false, - ImmutableMap.of( - "book", - ReservedEntry.create(ReservationType.ALLOWED_IN_SUNRISE, null), - "music", - ReservedEntry.create( - ReservationType.RESERVED_FOR_ANCHOR_TENANT, "reserved for anchor tenant"))); - - assertThat(reservedList.getName()).isEqualTo("app"); - assertThat(reservedList.getShouldPublish()).isFalse(); - assertThat(reservedList.getLabelsToReservations()) - .containsExactly( - "book", - ReservedEntry.create(ReservationType.ALLOWED_IN_SUNRISE, null), - "music", - ReservedEntry.create( - ReservationType.RESERVED_FOR_ANCHOR_TENANT, "reserved for anchor tenant")); - } - - @Test - public void create_throwsExceptionWhenLabelIsNotLowercase() { - Exception e = - assertThrows( - IllegalArgumentException.class, - () -> - ReservedList.create( - "UPPER.tld", - true, - ImmutableMap.of("UPPER", ReservedEntry.create(FULLY_BLOCKED, "")))); - assertThat(e) - .hasMessageThat() - .contains("Label(s) [UPPER] must be in puny-coded, lower-case form"); - } - - @Test - public void create_labelMustBePunyCoded() { - Exception e = - assertThrows( - IllegalArgumentException.class, - () -> - ReservedList.create( - "lower.みんな", - true, - ImmutableMap.of("みんな", ReservedEntry.create(FULLY_BLOCKED, "")))); - assertThat(e) - .hasMessageThat() - .contains("Label(s) [みんな] must be in puny-coded, lower-case form"); - } -} diff --git a/core/src/test/java/google/registry/schema/tmch/ClaimsListDaoTest.java b/core/src/test/java/google/registry/schema/tmch/ClaimsListDaoTest.java index 368c75818..c1f5ab839 100644 --- a/core/src/test/java/google/registry/schema/tmch/ClaimsListDaoTest.java +++ b/core/src/test/java/google/registry/schema/tmch/ClaimsListDaoTest.java @@ -19,7 +19,9 @@ import static com.google.common.truth.Truth.assertThat; import com.google.common.collect.ImmutableMap; import google.registry.persistence.transaction.JpaTestRules; import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageExtension; +import google.registry.testing.DatastoreEntityExtension; import google.registry.testing.FakeClock; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -32,6 +34,10 @@ public class ClaimsListDaoTest { public final JpaIntegrationWithCoverageExtension jpa = new JpaTestRules.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension(); + @RegisterExtension + @Order(value = 1) + public final DatastoreEntityExtension datastoreEntityExtension = new DatastoreEntityExtension(); + @Test public void trySave_insertsClaimsListSuccessfully() { ClaimsList claimsList = diff --git a/core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTest.java b/core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTest.java deleted file mode 100644 index 71369b259..000000000 --- a/core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTest.java +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2019 The Nomulus Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package google.registry.tools; - -import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.registry.label.ReservationType.ALLOWED_IN_SUNRISE; -import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED; -import static google.registry.tools.CreateOrUpdateReservedListCommand.parseToReservationsByLabels; -import static org.junit.Assert.assertThrows; - -import com.google.common.collect.ImmutableList; -import google.registry.model.registry.label.ReservationType; -import google.registry.schema.tld.ReservedList.ReservedEntry; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link CreateOrUpdateReservedListCommand}. */ -@RunWith(JUnit4.class) -public class CreateOrUpdateReservedListCommandTest { - - @Test - public void parseToReservationsByLabels_worksCorrectly() { - assertThat( - parseToReservationsByLabels( - ImmutableList.of( - "reserveddomain,FULLY_BLOCKED", - "availableinga,ALLOWED_IN_SUNRISE#allowed_in_sunrise", - "fourletterword,FULLY_BLOCKED"))) - .containsExactly( - "reserveddomain", - ReservedEntry.create(ReservationType.FULLY_BLOCKED, ""), - "availableinga", - ReservedEntry.create(ALLOWED_IN_SUNRISE, "allowed_in_sunrise"), - "fourletterword", - ReservedEntry.create(FULLY_BLOCKED, "")); - } - - @Test - public void parseToReservationsByLabels_throwsExceptionForInvalidLabel() { - Throwable thrown = - assertThrows( - IllegalArgumentException.class, - () -> - parseToReservationsByLabels( - ImmutableList.of("reserveddomain,FULLY_BLOCKED", "UPPER,FULLY_BLOCKED"))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo("Label 'UPPER' must be in puny-coded, lower-case form"); - } - - @Test - public void parseToReservationsByLabels_throwsExceptionForNonPunyCodedLabel() { - Throwable thrown = - assertThrows( - IllegalArgumentException.class, - () -> - parseToReservationsByLabels( - ImmutableList.of("reserveddomain,FULLY_BLOCKED", "みんな,FULLY_BLOCKED"))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo("Label 'みんな' must be in puny-coded, lower-case form"); - } - - @Test - public void parseToReservationsByLabels_throwsExceptionForInvalidReservationType() { - Throwable thrown = - assertThrows( - IllegalArgumentException.class, - () -> - parseToReservationsByLabels( - ImmutableList.of( - "reserveddomain,FULLY_BLOCKED", "invalidtype,INVALID_RESERVATION_TYPE"))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "No enum constant" - + " google.registry.model.registry.label.ReservationType.INVALID_RESERVATION_TYPE"); - } - - @Test - public void parseToReservationsByLabels_throwsExceptionForInvalidLines() { - Throwable thrown = - assertThrows( - IllegalArgumentException.class, - () -> - parseToReservationsByLabels( - ImmutableList.of( - "reserveddomain,FULLY_BLOCKED,too,many,parts", - "fourletterword,FULLY_BLOCKED"))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "Could not parse line in reserved list: reserveddomain,FULLY_BLOCKED,too,many,parts"); - } - - @Test - public void parseToReservationsByLabels_throwsExceptionForDuplicateEntries() { - Throwable thrown = - assertThrows( - IllegalStateException.class, - () -> - parseToReservationsByLabels( - ImmutableList.of( - "reserveddomain,FULLY_BLOCKED", - "fourletterword,FULLY_BLOCKED", - "fourletterword,FULLY_BLOCKED"))); - assertThat(thrown) - .hasMessageThat() - .isEqualTo( - "Reserved list cannot contain duplicate labels. Dupes (with counts) were:" - + " [fourletterword x 2]"); - } -} diff --git a/core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTestCase.java b/core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTestCase.java index 38b56c0bc..5f6c3da0a 100644 --- a/core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTestCase.java +++ b/core/src/test/java/google/registry/tools/CreateOrUpdateReservedListCommandTestCase.java @@ -22,15 +22,16 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertThrows; import com.beust.jcommander.ParameterException; +import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; import com.google.common.truth.Truth8; import google.registry.model.registry.label.ReservedList; -import google.registry.schema.tld.ReservedList.ReservedEntry; -import google.registry.schema.tld.ReservedListDao; +import google.registry.model.registry.label.ReservedList.ReservedListEntry; +import google.registry.model.registry.label.ReservedListSqlDao; import java.io.File; import java.io.IOException; -import java.util.Map; import javax.persistence.EntityManager; +import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; @@ -100,12 +101,20 @@ public abstract class CreateOrUpdateReservedListCommandTestCase< .isEqualTo("Label example.tld must not be a multi-level domain name"); } - google.registry.schema.tld.ReservedList createCloudSqlReservedList( - String name, boolean shouldPublish, Map labelsToEntries) { - return google.registry.schema.tld.ReservedList.create(name, shouldPublish, labelsToEntries); + ReservedList createCloudSqlReservedList( + String name, + DateTime creationTime, + boolean shouldPublish, + ImmutableMap labelsToEntries) { + return new ReservedList.Builder() + .setName(name) + .setLastUpdateTime(creationTime) + .setShouldPublish(shouldPublish) + .setReservedListMap(labelsToEntries) + .build(); } - google.registry.schema.tld.ReservedList getCloudSqlReservedList(String name) { + ReservedList getCloudSqlReservedList(String name) { return jpaTm() .transact( () -> { @@ -117,27 +126,25 @@ public abstract class CreateOrUpdateReservedListCommandTestCase< .setParameter("name", name) .getSingleResult(); return em.createQuery( - "FROM ReservedList rl LEFT JOIN FETCH rl.labelsToReservations WHERE" + "FROM ReservedList rl LEFT JOIN FETCH rl.reservedListMap WHERE" + " rl.revisionId = :revisionId", - google.registry.schema.tld.ReservedList.class) + ReservedList.class) .setParameter("revisionId", revisionId) .getSingleResult(); }); } void verifyXnq9jyb4cInCloudSql() { - assertThat(ReservedListDao.checkExists("xn--q9jyb4c_common-reserved")).isTrue(); - google.registry.schema.tld.ReservedList persistedList = - getCloudSqlReservedList("xn--q9jyb4c_common-reserved"); + assertThat(ReservedListSqlDao.checkExists("xn--q9jyb4c_common-reserved")).isTrue(); + ReservedList persistedList = getCloudSqlReservedList("xn--q9jyb4c_common-reserved"); assertThat(persistedList.getName()).isEqualTo("xn--q9jyb4c_common-reserved"); assertThat(persistedList.getShouldPublish()).isTrue(); - assertThat(persistedList.getCreationTimestamp()).isEqualTo(fakeClock.nowUtc()); - assertThat(persistedList.getLabelsToReservations()) + assertThat(persistedList.getReservedListEntries()) .containsExactly( "baddies", - ReservedEntry.create(FULLY_BLOCKED, ""), + ReservedListEntry.create("baddies", FULLY_BLOCKED, ""), "ford", - ReservedEntry.create(FULLY_BLOCKED, "random comment")); + ReservedListEntry.create("ford", FULLY_BLOCKED, "random comment")); } void verifyXnq9jyb4cInDatastore() { diff --git a/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java b/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java index fc9ff7bd6..e79f3180a 100644 --- a/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java +++ b/core/src/test/java/google/registry/tools/CreateReservedListCommandTest.java @@ -27,8 +27,8 @@ import static org.junit.Assert.assertThrows; import com.google.common.collect.ImmutableMap; import google.registry.model.registry.Registry; import google.registry.model.registry.label.ReservedList; -import google.registry.schema.tld.ReservedList.ReservedEntry; -import google.registry.schema.tld.ReservedListDao; +import google.registry.model.registry.label.ReservedList.ReservedListEntry; +import google.registry.model.registry.label.ReservedListSqlDao; import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; @@ -187,11 +187,13 @@ public class CreateReservedListCommandTest extends public void testSaveToCloudSql_noExceptionThrownWhenSaveFail() throws Exception { // Note that, during the dual-write phase, we want to make sure that no exception will be // thrown if saving reserved list to Cloud SQL fails. - ReservedListDao.save( + ReservedListSqlDao.save( createCloudSqlReservedList( "xn--q9jyb4c_common-reserved", + fakeClock.nowUtc(), true, - ImmutableMap.of("testdomain", ReservedEntry.create(FULLY_BLOCKED, "")))); + ImmutableMap.of( + "testdomain", ReservedListEntry.create("testdomain", FULLY_BLOCKED, "")))); runCommandForced("--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath); verifyXnq9jyb4cInDatastore(); } diff --git a/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java b/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java index 5b7139ae5..fe5f30ecb 100644 --- a/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java +++ b/core/src/test/java/google/registry/tools/UpdateReservedListCommandTest.java @@ -24,8 +24,8 @@ import static org.junit.Assert.assertThrows; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import google.registry.model.registry.label.ReservedList; -import google.registry.schema.tld.ReservedList.ReservedEntry; -import google.registry.schema.tld.ReservedListDao; +import google.registry.model.registry.label.ReservedList.ReservedListEntry; +import google.registry.model.registry.label.ReservedListSqlDao; import org.junit.Before; import org.junit.Test; @@ -50,11 +50,13 @@ public class UpdateReservedListCommandTest extends } private void populateInitialReservedListInCloudSql(boolean shouldPublish) { - ReservedListDao.save( + ReservedListSqlDao.save( createCloudSqlReservedList( "xn--q9jyb4c_common-reserved", + fakeClock.nowUtc(), shouldPublish, - ImmutableMap.of("helicopter", ReservedEntry.create(FULLY_BLOCKED, "")))); + ImmutableMap.of( + "helicopter", ReservedListEntry.create("helicopter", FULLY_BLOCKED, "")))); } @Test @@ -131,6 +133,6 @@ public class UpdateReservedListCommandTest extends // Datastore when we update it. runCommandForced("--name=xn--q9jyb4c_common-reserved", "--input=" + reservedTermsPath); verifyXnq9jyb4cInDatastore(); - assertThat(ReservedListDao.checkExists("xn--q9jyb4c_common-reserved")).isTrue(); + assertThat(ReservedListSqlDao.checkExists("xn--q9jyb4c_common-reserved")).isTrue(); } } diff --git a/db/src/main/resources/sql/schema/db-schema.sql.generated b/db/src/main/resources/sql/schema/db-schema.sql.generated index d48ad7541..2ed7bea5e 100644 --- a/db/src/main/resources/sql/schema/db-schema.sql.generated +++ b/db/src/main/resources/sql/schema/db-schema.sql.generated @@ -462,8 +462,8 @@ create sequence history_id_sequence start 1 increment 1; create table "ReservedEntry" ( revision_id int8 not null, - comment text, reservation_type int4 not null, + comment text, domain_label text not null, primary key (revision_id, domain_label) );