From c130cdb04225561fb19e5bd326ef8797ece63e5d Mon Sep 17 00:00:00 2001 From: Ben McIlwain Date: Wed, 9 Oct 2019 17:06:42 -0400 Subject: [PATCH] Add Bloom filters to the Cloud SQL PremiumList schema (#306) * Add Bloom filters to the Cloud SQL PremiumList schema They are slightly different from the existing Bloom filters stored in Datastore in that they now use an ASCII String encoding rather than the more generic CharSequence, and there is no maximum size (whereas we previously had to live within the 1 MB max entity size for Datastore). --- .../persistence/BloomFilterConverter.java | 59 ++++++++++++++++ .../registry/schema/tld/PremiumList.java | 26 ++++++- .../main/resources/META-INF/persistence.xml | 1 + .../persistence/BloomFilterConverterTest.java | 67 +++++++++++++++++++ .../registry/schema/tld/PremiumListTest.java | 50 ++++++++++++++ .../flyway/V6__premium_list_bloom_filter.sql | 15 +++++ .../sql/schema/db-schema.sql.generated | 1 + .../resources/sql/schema/nomulus.golden.sql | 3 +- 8 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/google/registry/persistence/BloomFilterConverter.java create mode 100644 core/src/test/java/google/registry/persistence/BloomFilterConverterTest.java create mode 100644 core/src/test/java/google/registry/schema/tld/PremiumListTest.java create mode 100644 db/src/main/resources/sql/flyway/V6__premium_list_bloom_filter.sql diff --git a/core/src/main/java/google/registry/persistence/BloomFilterConverter.java b/core/src/main/java/google/registry/persistence/BloomFilterConverter.java new file mode 100644 index 000000000..3622ce68b --- /dev/null +++ b/core/src/main/java/google/registry/persistence/BloomFilterConverter.java @@ -0,0 +1,59 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package google.registry.persistence; + +import static com.google.common.base.Charsets.US_ASCII; +import static com.google.common.hash.Funnels.stringFunnel; + +import com.google.common.hash.BloomFilter; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import javax.annotation.Nullable; +import javax.persistence.AttributeConverter; +import javax.persistence.Converter; + +/** JPA converter for ASCII String {@link BloomFilter}s. */ +@Converter(autoApply = true) +public class BloomFilterConverter implements AttributeConverter, byte[]> { + + @Override + @Nullable + public byte[] convertToDatabaseColumn(@Nullable BloomFilter entity) { + if (entity == null) { + return null; + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try { + entity.writeTo(bos); + } catch (IOException e) { + throw new UncheckedIOException("Error saving Bloom filter data", e); + } + return bos.toByteArray(); + } + + @Override + @Nullable + public BloomFilter convertToEntityAttribute(@Nullable byte[] columnValue) { + if (columnValue == null) { + return null; + } + try { + return BloomFilter.readFrom(new ByteArrayInputStream(columnValue), stringFunnel(US_ASCII)); + } catch (IOException e) { + throw new UncheckedIOException("Error loading Bloom filter data", e); + } + } +} diff --git a/core/src/main/java/google/registry/schema/tld/PremiumList.java b/core/src/main/java/google/registry/schema/tld/PremiumList.java index 7b0de3cb2..ad31ee75c 100644 --- a/core/src/main/java/google/registry/schema/tld/PremiumList.java +++ b/core/src/main/java/google/registry/schema/tld/PremiumList.java @@ -14,8 +14,12 @@ package google.registry.schema.tld; +import static com.google.common.base.Charsets.US_ASCII; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.hash.Funnels.stringFunnel; +import com.google.common.collect.ImmutableMap; +import com.google.common.hash.BloomFilter; import google.registry.model.CreateAutoTimestamp; import java.math.BigDecimal; import java.util.Map; @@ -67,11 +71,16 @@ public class PremiumList { @Column(name = "price", nullable = false) private Map labelsToPrices; + @Column(nullable = false) + private BloomFilter bloomFilter; + private PremiumList(String name, CurrencyUnit currency, Map labelsToPrices) { - // TODO(mcilwain): Generate the Bloom filter and set it here. this.name = name; this.currency = currency; this.labelsToPrices = labelsToPrices; + // ASCII is used for the charset because all premium list domain labels are stored punycoded. + this.bloomFilter = BloomFilter.create(stringFunnel(US_ASCII), labelsToPrices.size()); + labelsToPrices.keySet().forEach(this.bloomFilter::put); } // Hibernate requires this default constructor. @@ -101,7 +110,18 @@ public class PremiumList { } /** Returns a {@link Map} of domain labels to prices. */ - public Map getLabelsToPrices() { - return labelsToPrices; + public ImmutableMap getLabelsToPrices() { + return ImmutableMap.copyOf(labelsToPrices); + } + + /** + * Returns a Bloom filter to determine whether a label might be premium, or is definitely not. + * + *

If the domain label might be premium, then the next step is to check for the existence of a + * corresponding row in the PremiumListEntry table. Otherwise, we know for sure it's not premium, + * and no DB load is required. + */ + public BloomFilter getBloomFilter() { + return bloomFilter; } } diff --git a/core/src/main/resources/META-INF/persistence.xml b/core/src/main/resources/META-INF/persistence.xml index 705541169..e7393ed1c 100644 --- a/core/src/main/resources/META-INF/persistence.xml +++ b/core/src/main/resources/META-INF/persistence.xml @@ -33,6 +33,7 @@ google.registry.model.eppcommon.Trid + google.registry.persistence.BloomFilterConverter google.registry.persistence.CreateAutoTimestampConverter google.registry.persistence.UpdateAutoTimestampConverter google.registry.persistence.ZonedDateTimeConverter diff --git a/core/src/test/java/google/registry/persistence/BloomFilterConverterTest.java b/core/src/test/java/google/registry/persistence/BloomFilterConverterTest.java new file mode 100644 index 000000000..d1675a391 --- /dev/null +++ b/core/src/test/java/google/registry/persistence/BloomFilterConverterTest.java @@ -0,0 +1,67 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package google.registry.persistence; + +import static com.google.common.base.Charsets.US_ASCII; +import static com.google.common.hash.Funnels.stringFunnel; +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.transaction.TransactionManagerFactory.jpaTm; + +import com.google.common.collect.ImmutableSet; +import com.google.common.hash.BloomFilter; +import google.registry.model.ImmutableObject; +import google.registry.model.transaction.JpaTransactionManagerRule; +import javax.persistence.Entity; +import javax.persistence.Id; +import org.hibernate.cfg.Environment; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BloomFilterConverterTest { + + @Rule + public final JpaTransactionManagerRule jpaTmRule = + new JpaTransactionManagerRule.Builder() + .withEntityClass(TestEntity.class) + .withProperty(Environment.HBM2DDL_AUTO, "update") + .build(); + + @Test + public void roundTripConversion_returnsSameBloomFilter() { + BloomFilter bloomFilter = BloomFilter.create(stringFunnel(US_ASCII), 3); + ImmutableSet.of("foo", "bar", "baz").forEach(bloomFilter::put); + TestEntity entity = new TestEntity(bloomFilter); + jpaTm().transact(() -> jpaTm().getEntityManager().persist(entity)); + TestEntity persisted = + jpaTm().transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "id")); + assertThat(persisted.bloomFilter).isEqualTo(bloomFilter); + } + + @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference. + public static class TestEntity extends ImmutableObject { + + @Id String name = "id"; + + BloomFilter bloomFilter; + + public TestEntity() {} + + public TestEntity(BloomFilter bloomFilter) { + this.bloomFilter = bloomFilter; + } + } +} diff --git a/core/src/test/java/google/registry/schema/tld/PremiumListTest.java b/core/src/test/java/google/registry/schema/tld/PremiumListTest.java new file mode 100644 index 000000000..831718d3a --- /dev/null +++ b/core/src/test/java/google/registry/schema/tld/PremiumListTest.java @@ -0,0 +1,50 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.schema.tld; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.hash.BloomFilter; +import java.math.BigDecimal; +import org.joda.money.CurrencyUnit; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link PremiumList}. */ +@RunWith(JUnit4.class) +public class PremiumListTest { + + private static final ImmutableMap TEST_PRICES = + ImmutableMap.of( + "silver", + BigDecimal.valueOf(10.23), + "gold", + BigDecimal.valueOf(1305.47), + "palladium", + BigDecimal.valueOf(1552.78)); + + @Test + public void bloomFilter_worksCorrectly() { + BloomFilter bloomFilter = + PremiumList.create("testname", CurrencyUnit.USD, TEST_PRICES).getBloomFilter(); + ImmutableSet.of("silver", "gold", "palladium") + .forEach(l -> assertThat(bloomFilter.mightContain(l)).isTrue()); + ImmutableSet.of("dirt", "pyrite", "zirconia") + .forEach(l -> assertThat(bloomFilter.mightContain(l)).isFalse()); + } +} diff --git a/db/src/main/resources/sql/flyway/V6__premium_list_bloom_filter.sql b/db/src/main/resources/sql/flyway/V6__premium_list_bloom_filter.sql new file mode 100644 index 000000000..2529f3df3 --- /dev/null +++ b/db/src/main/resources/sql/flyway/V6__premium_list_bloom_filter.sql @@ -0,0 +1,15 @@ +-- Copyright 2019 The Nomulus Authors. All Rights Reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +alter table "PremiumList" add column if not exists bloom_filter bytea not null; diff --git a/db/src/main/resources/sql/schema/db-schema.sql.generated b/db/src/main/resources/sql/schema/db-schema.sql.generated index 982ca1ea4..615b4e734 100644 --- a/db/src/main/resources/sql/schema/db-schema.sql.generated +++ b/db/src/main/resources/sql/schema/db-schema.sql.generated @@ -130,6 +130,7 @@ create table "PremiumList" ( revision_id bigserial not null, + bloom_filter bytea not null, creation_timestamp timestamptz not null, currency bytea not null, name text not null, diff --git a/db/src/main/resources/sql/schema/nomulus.golden.sql b/db/src/main/resources/sql/schema/nomulus.golden.sql index df7ec8e4c..ba78f3b34 100644 --- a/db/src/main/resources/sql/schema/nomulus.golden.sql +++ b/db/src/main/resources/sql/schema/nomulus.golden.sql @@ -92,7 +92,8 @@ CREATE TABLE public."PremiumList" ( revision_id bigint NOT NULL, creation_timestamp timestamp with time zone NOT NULL, currency bytea NOT NULL, - name text NOT NULL + name text NOT NULL, + bloom_filter bytea NOT NULL );