diff --git a/java/google/registry/config/RegistryConfig.java b/java/google/registry/config/RegistryConfig.java index 3fabed129..802ae4fd2 100644 --- a/java/google/registry/config/RegistryConfig.java +++ b/java/google/registry/config/RegistryConfig.java @@ -1057,6 +1057,13 @@ public final class RegistryConfig { return config.registryPolicy.allocationTokenCustomLogicClass; } + /** Returns the disclaimer text for the exported premium terms. */ + @Provides + @Config("premiumTermsExportDisclaimer") + public static String providePremiumTermsExportDisclaimer(RegistryConfigSettings config) { + return formatComments(config.registryPolicy.reservedTermsExportDisclaimer); + } + /** * Returns the header text at the top of the reserved terms exported list. * @@ -1065,13 +1072,7 @@ public final class RegistryConfig { @Provides @Config("reservedTermsExportDisclaimer") public static String provideReservedTermsExportDisclaimer(RegistryConfigSettings config) { - return Splitter.on('\n') - .omitEmptyStrings() - .trimResults() - .splitToList(config.registryPolicy.reservedTermsExportDisclaimer) - .stream() - .map(s -> "# " + s) - .collect(Collectors.joining("\n")); + return formatComments(config.registryPolicy.reservedTermsExportDisclaimer); } /** Returns the clientId of the registrar used by the {@code CheckApiServlet}. */ @@ -1412,5 +1413,11 @@ public final class RegistryConfig { static final Supplier CONFIG_SETTINGS = memoize(YamlUtils::getConfigSettings); + private static String formatComments(String text) { + return Splitter.on('\n').omitEmptyStrings().trimResults().splitToList(text).stream() + .map(s -> "# " + s) + .collect(Collectors.joining("\n")); + } + private RegistryConfig() {} } diff --git a/java/google/registry/config/RegistryConfigSettings.java b/java/google/registry/config/RegistryConfigSettings.java index 35c64ff85..0aeabba1d 100644 --- a/java/google/registry/config/RegistryConfigSettings.java +++ b/java/google/registry/config/RegistryConfigSettings.java @@ -87,6 +87,7 @@ public class RegistryConfigSettings { public String tmchMarksDbUrl; public String checkApiServletClientId; public String registryAdminClientId; + public String premiumTermsExportDisclaimer; public String reservedTermsExportDisclaimer; public String whoisDisclaimer; } diff --git a/java/google/registry/config/files/default-config.yaml b/java/google/registry/config/files/default-config.yaml index 7efc39452..379ee0eaa 100644 --- a/java/google/registry/config/files/default-config.yaml +++ b/java/google/registry/config/files/default-config.yaml @@ -77,6 +77,12 @@ registryPolicy: # registrar registryAdminClientId: TheRegistrar + # Disclaimer at the top of the exported premium terms list. + premiumTermsExportDisclaimer: | + This list contains domains for the TLD offered at a premium price. This + list is subject to change. The most up-to-date source is always the + registry itself, by sending domain check EPP commands. + # Disclaimer at the top of the exported reserved terms list. reservedTermsExportDisclaimer: | This list contains reserved terms for the TLD. Other terms may be reserved diff --git a/java/google/registry/env/alpha/default/WEB-INF/cron.xml b/java/google/registry/env/alpha/default/WEB-INF/cron.xml index 223da547b..086cba362 100644 --- a/java/google/registry/env/alpha/default/WEB-INF/cron.xml +++ b/java/google/registry/env/alpha/default/WEB-INF/cron.xml @@ -178,6 +178,15 @@ backend + + + + Premium terms export to Google Drive job for creating once-daily exports. + + every day 05:00 + backend + + diff --git a/java/google/registry/env/common/backend/WEB-INF/web.xml b/java/google/registry/env/common/backend/WEB-INF/web.xml index 17afdfc93..714c469eb 100644 --- a/java/google/registry/env/common/backend/WEB-INF/web.xml +++ b/java/google/registry/env/common/backend/WEB-INF/web.xml @@ -280,6 +280,12 @@ /_dr/task/syncRegistrarsSheet + + + backend-servlet + /_dr/task/exportPremiumTerms + + backend-servlet diff --git a/java/google/registry/export/ExportPremiumTermsAction.java b/java/google/registry/export/ExportPremiumTermsAction.java new file mode 100644 index 000000000..db0e33e38 --- /dev/null +++ b/java/google/registry/export/ExportPremiumTermsAction.java @@ -0,0 +1,150 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static google.registry.model.registry.label.PremiumListUtils.loadPremiumListEntries; +import static google.registry.request.Action.Method.POST; +import static java.nio.charset.StandardCharsets.UTF_8; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Streams; +import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; +import google.registry.config.RegistryConfig.Config; +import google.registry.model.registry.Registry; +import google.registry.model.registry.label.PremiumList; +import google.registry.request.Action; +import google.registry.request.Parameter; +import google.registry.request.RequestParameters; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.storage.drive.DriveConnection; +import java.io.IOException; +import java.util.Optional; +import java.util.SortedSet; +import javax.inject.Inject; + +/** Action that exports the premium terms list for a TLD to Google Drive. */ +@Action(path = "/_dr/task/exportPremiumTerms", method = POST, auth = Auth.AUTH_INTERNAL_ONLY) +public class ExportPremiumTermsAction implements Runnable { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + static final MediaType EXPORT_MIME_TYPE = MediaType.PLAIN_TEXT_UTF_8; + static final String PREMIUM_TERMS_FILENAME = "CONFIDENTIAL_premium_terms.txt"; + + @Inject DriveConnection driveConnection; + + @Inject + @Config("premiumTermsExportDisclaimer") + String exportDisclaimer; + + @Inject @Parameter(RequestParameters.PARAM_TLD) String tld; + @Inject Response response; + + @Inject + ExportPremiumTermsAction() {} + + /** + * Exports the premium terms for the TLD specified via the "tld" param to a file in the Google + * Drive folder configured for that TLD. + * + *

The export file is named "CONFIDENTIAL_premium_terms.txt" and is encoded in UTF-8. It begins + * with the disclaimer text that is immediately followed by premium terms, each occupying a line. + * The file ends with a trailing newline. + * + *

Each term is formatted as "term,price", where price is the ISO-4217 three-letter currency + * code followed by a space and then the numeric amount. For example: + * + * 

+   * bank,USD 1599.00
+   *
+ * + *

This servlet prints the ID of the file in GoogleDrive that was created/updated. + */ + @Override + public void run() { + response.setContentType(PLAIN_TEXT_UTF_8); + try { + Registry registry = Registry.get(tld); + String resultMsg = checkConfig(registry).orElseGet(() -> exportPremiumTerms(registry)); + response.setStatus(SC_OK); + response.setPayload(resultMsg); + } catch (Throwable e) { + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setPayload(e.getMessage()); + throw new RuntimeException( + String.format("Exception occurred while exporting premium terms for TLD %s.", tld), e); + } + } + + /** + * Checks if {@code registry} is properly configured to export premium terms. + * + * @return {@link Optional#empty()} if {@code registry} export may proceed. Otherwise returns an + * error message + */ + private Optional checkConfig(Registry registry) { + if (isNullOrEmpty(registry.getDriveFolderId())) { + logger.atInfo().log( + "Skipping premium terms export for TLD %s because Drive folder isn't specified", tld); + return Optional.of("Skipping export because no Drive folder is associated with this TLD"); + } + if (registry.getPremiumList() == null) { + logger.atInfo().log("No premium terms to export for TLD %s", tld); + return Optional.of("No premium lists configured"); + } + return Optional.empty(); + } + + private String exportPremiumTerms(Registry registry) { + try { + String fileId = + driveConnection.createOrUpdateFile( + PREMIUM_TERMS_FILENAME, + EXPORT_MIME_TYPE, + registry.getDriveFolderId(), + getFormattedPremiumTerms(registry).getBytes(UTF_8)); + logger.atInfo().log( + "Exporting premium terms succeeded for TLD %s, file ID is: %s", tld, fileId); + return fileId; + } catch (IOException e) { + throw new RuntimeException("Error exporting premium terms file to Drive.", e); + } + } + + private String getFormattedPremiumTerms(Registry registry) { + Optional premiumList = PremiumList.getCached(registry.getPremiumList().getName()); + checkState(premiumList.isPresent(), "Could not load premium list for " + tld); + SortedSet premiumTerms = + Streams.stream(loadPremiumListEntries(premiumList.get())) + .map(entry -> Joiner.on(",").join(entry.getLabel(), entry.getValue())) + .collect(ImmutableSortedSet.toImmutableSortedSet(String::compareTo)); + + return Joiner.on("\n") + .appendTo( + new StringBuilder(), + Iterables.concat(ImmutableList.of(exportDisclaimer.trim()), premiumTerms)) + .append("\n") + .toString(); + } +} diff --git a/java/google/registry/model/registry/label/PremiumListUtils.java b/java/google/registry/model/registry/label/PremiumListUtils.java index eb5c0c46b..e9c1515d7 100644 --- a/java/google/registry/model/registry/label/PremiumListUtils.java +++ b/java/google/registry/model/registry/label/PremiumListUtils.java @@ -213,6 +213,15 @@ public final class PremiumListUtils { ofy().transactNew(() -> ofy().delete().key(premiumList.getRevisionKey())); } + /** + * Returns all {@link PremiumListEntry PremiumListEntries} in the given {@code premiumList}. + * + *

This is an expensive operation and should only be used when the entire list is required. + */ + public static Iterable loadPremiumListEntries(PremiumList premiumList) { + return ofy().load().type(PremiumListEntry.class).ancestor(premiumList.revisionKey).iterable(); + } + /** Returns whether a PremiumList of the given name exists, bypassing the cache. */ public static boolean doesPremiumListExist(String name) { return ofy().load().key(Key.create(getCrossTldKey(), PremiumList.class, name)).now() != null; diff --git a/java/google/registry/module/backend/BackendRequestComponent.java b/java/google/registry/module/backend/BackendRequestComponent.java index baba1f7fa..a36d57b0c 100644 --- a/java/google/registry/module/backend/BackendRequestComponent.java +++ b/java/google/registry/module/backend/BackendRequestComponent.java @@ -44,6 +44,7 @@ import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule; import google.registry.export.BigqueryPollJobAction; import google.registry.export.CheckSnapshotAction; import google.registry.export.ExportDomainListsAction; +import google.registry.export.ExportPremiumTermsAction; import google.registry.export.ExportRequestModule; import google.registry.export.ExportReservedTermsAction; import google.registry.export.ExportSnapshotAction; @@ -129,6 +130,7 @@ interface BackendRequestComponent { ExpandRecurringBillingEventsAction expandRecurringBillingEventsAction(); ExportCommitLogDiffAction exportCommitLogDiffAction(); ExportDomainListsAction exportDomainListsAction(); + ExportPremiumTermsAction exportPremiumTermsAction(); ExportReservedTermsAction exportReservedTermsAction(); ExportSnapshotAction exportSnapshotAction(); GenerateInvoicesAction generateInvoicesAction(); diff --git a/javatests/google/registry/export/ExportPremiumTermsActionTest.java b/javatests/google/registry/export/ExportPremiumTermsActionTest.java new file mode 100644 index 000000000..819039676 --- /dev/null +++ b/javatests/google/registry/export/ExportPremiumTermsActionTest.java @@ -0,0 +1,195 @@ +// Copyright 2018 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static google.registry.export.ExportPremiumTermsAction.EXPORT_MIME_TYPE; +import static google.registry.export.ExportPremiumTermsAction.PREMIUM_TERMS_FILENAME; +import static google.registry.model.registry.label.PremiumListUtils.deletePremiumList; +import static google.registry.model.registry.label.PremiumListUtils.savePremiumListAndEntries; +import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.DatastoreHelper.deleteTld; +import static google.registry.testing.DatastoreHelper.persistResource; +import static google.registry.testing.JUnitBackports.assertThrows; +import static java.nio.charset.StandardCharsets.UTF_8; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_OK; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.net.MediaType; +import google.registry.model.registry.Registry; +import google.registry.model.registry.label.PremiumList; +import google.registry.request.Response; +import google.registry.storage.drive.DriveConnection; +import google.registry.testing.AppEngineRule; +import java.io.IOException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Matchers; + +@RunWith(JUnit4.class) +public class ExportPremiumTermsActionTest { + + private static final String DISCLAIMER_WITH_NEWLINE = "# Premium Terms Export Disclaimer\n"; + private static final ImmutableList PREMIUM_NAMES = + ImmutableList.of("2048,USD 549", "0,USD 549"); + private static final String EXPECTED_FILE_CONTENT = + DISCLAIMER_WITH_NEWLINE + "0,USD 549.00\n" + "2048,USD 549.00\n"; + + @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build(); + + private final DriveConnection driveConnection = mock(DriveConnection.class); + private final Response response = mock(Response.class); + + private void runAction(String tld) { + ExportPremiumTermsAction action = new ExportPremiumTermsAction(); + action.response = response; + action.driveConnection = driveConnection; + action.exportDisclaimer = DISCLAIMER_WITH_NEWLINE; + action.tld = tld; + action.run(); + } + + @Before + public void setup() throws Exception { + createTld("tld"); + PremiumList pl = new PremiumList.Builder().setName("pl-name").build(); + savePremiumListAndEntries(pl, PREMIUM_NAMES); + persistResource( + Registry.get("tld").asBuilder().setPremiumList(pl).setDriveFolderId("folder_id").build()); + when(driveConnection.createOrUpdateFile( + anyString(), any(MediaType.class), eq("folder_id"), any(byte[].class))) + .thenReturn("file_id"); + when(driveConnection.createOrUpdateFile( + anyString(), any(MediaType.class), eq("bad_folder_id"), any(byte[].class))) + .thenThrow(new IOException()); + } + + @Test + public void test_exportPremiumTerms_success() throws IOException { + runAction("tld"); + + verify(driveConnection) + .createOrUpdateFile( + PREMIUM_TERMS_FILENAME, + EXPORT_MIME_TYPE, + "folder_id", + EXPECTED_FILE_CONTENT.getBytes(UTF_8)); + verifyNoMoreInteractions(driveConnection); + + verify(response).setStatus(SC_OK); + verify(response).setPayload("file_id"); + verify(response).setContentType(PLAIN_TEXT_UTF_8); + verifyNoMoreInteractions(response); + } + + @Test + public void test_exportPremiumTerms_success_emptyPremiumList() throws IOException { + PremiumList pl = new PremiumList.Builder().setName("pl-name").build(); + savePremiumListAndEntries(pl, ImmutableList.of()); + runAction("tld"); + + verify(driveConnection) + .createOrUpdateFile( + PREMIUM_TERMS_FILENAME, + EXPORT_MIME_TYPE, + "folder_id", + DISCLAIMER_WITH_NEWLINE.getBytes(UTF_8)); + verifyNoMoreInteractions(driveConnection); + + verify(response).setStatus(SC_OK); + verify(response).setPayload("file_id"); + verify(response).setContentType(PLAIN_TEXT_UTF_8); + verifyNoMoreInteractions(response); + } + + @Test + public void test_exportPremiumTerms_doNothing_listNotConfigured() { + persistResource(Registry.get("tld").asBuilder().setPremiumList(null).build()); + runAction("tld"); + + verifyZeroInteractions(driveConnection); + verify(response).setStatus(SC_OK); + verify(response).setPayload("No premium lists configured"); + verify(response).setContentType(PLAIN_TEXT_UTF_8); + verifyNoMoreInteractions(response); + } + + @Test + public void testExportPremiumTerms_doNothing_driveIdNotConfiguredInTld() { + persistResource(Registry.get("tld").asBuilder().setDriveFolderId(null).build()); + runAction("tld"); + + verifyZeroInteractions(driveConnection); + verify(response).setStatus(SC_OK); + verify(response) + .setPayload("Skipping export because no Drive folder is associated with this TLD"); + verify(response).setContentType(PLAIN_TEXT_UTF_8); + verifyNoMoreInteractions(response); + } + + @Test + public void test_exportPremiumTerms_failure_noSuchTld() { + deleteTld("tld"); + assertThrows(RuntimeException.class, () -> runAction("tld")); + + verifyZeroInteractions(driveConnection); + verify(response).setStatus(SC_INTERNAL_SERVER_ERROR); + verify(response).setPayload(anyString()); + verify(response).setContentType(PLAIN_TEXT_UTF_8); + verifyNoMoreInteractions(response); + } + + @Test + public void test_exportPremiumTerms_failure_noPremiumList() { + deletePremiumList(new PremiumList.Builder().setName("pl-name").build()); + assertThrows(RuntimeException.class, () -> runAction("tld")); + + verifyZeroInteractions(driveConnection); + verify(response).setStatus(SC_INTERNAL_SERVER_ERROR); + verify(response).setPayload("Could not load premium list for " + "tld"); + verify(response).setContentType(PLAIN_TEXT_UTF_8); + verifyNoMoreInteractions(response); + } + + @Test + public void testExportPremiumTerms_failure_driveIdThrowsException() throws IOException { + persistResource(Registry.get("tld").asBuilder().setDriveFolderId("bad_folder_id").build()); + assertThrows(RuntimeException.class, () -> runAction("tld")); + + verify(driveConnection) + .createOrUpdateFile( + PREMIUM_TERMS_FILENAME, + EXPORT_MIME_TYPE, + "bad_folder_id", + EXPECTED_FILE_CONTENT.getBytes(UTF_8)); + verifyNoMoreInteractions(driveConnection); + verify(response).setStatus(SC_INTERNAL_SERVER_ERROR); + verify(response).setPayload(Matchers.contains("Error exporting premium terms file to Drive.")); + verify(response).setContentType(PLAIN_TEXT_UTF_8); + verifyNoMoreInteractions(response); + } +} diff --git a/javatests/google/registry/module/backend/testdata/backend_routing.txt b/javatests/google/registry/module/backend/testdata/backend_routing.txt index f4c5cba98..c9731e33f 100644 --- a/javatests/google/registry/module/backend/testdata/backend_routing.txt +++ b/javatests/google/registry/module/backend/testdata/backend_routing.txt @@ -14,6 +14,7 @@ PATH CLASS METHOD /_dr/task/expandRecurringBillingEvents ExpandRecurringBillingEventsAction GET n INTERNAL APP IGNORED /_dr/task/exportCommitLogDiff ExportCommitLogDiffAction POST y INTERNAL APP IGNORED /_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL APP IGNORED +/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL APP IGNORED /_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL APP IGNORED /_dr/task/exportSnapshot ExportSnapshotAction POST y INTERNAL APP IGNORED /_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL APP IGNORED