Export Premium names to Drive

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=212509587

java/google/registry/config/RegistryConfig.java

@@ -1057,6 +1057,13 @@ public final class RegistryConfig {
       return config.registryPolicy.allocationTokenCustomLogicClass;
     }
 
+    /** Returns the disclaimer text for the exported premium terms. */
+    @Provides
+    @Config("premiumTermsExportDisclaimer")
+    public static String providePremiumTermsExportDisclaimer(RegistryConfigSettings config) {
+      return formatComments(config.registryPolicy.reservedTermsExportDisclaimer);
+    }
+
     /**
      * Returns the header text at the top of the reserved terms exported list.
      *
@@ -1065,13 +1072,7 @@ public final class RegistryConfig {
     @Provides
     @Config("reservedTermsExportDisclaimer")
     public static String provideReservedTermsExportDisclaimer(RegistryConfigSettings config) {
-      return Splitter.on('\n')
-          .omitEmptyStrings()
-          .trimResults()
-          .splitToList(config.registryPolicy.reservedTermsExportDisclaimer)
-          .stream()
-          .map(s -> "# " + s)
-          .collect(Collectors.joining("\n"));
+      return formatComments(config.registryPolicy.reservedTermsExportDisclaimer);
     }
 
     /** Returns the clientId of the registrar used by the {@code CheckApiServlet}. */
@@ -1412,5 +1413,11 @@ public final class RegistryConfig {
   static final Supplier<RegistryConfigSettings> CONFIG_SETTINGS =
       memoize(YamlUtils::getConfigSettings);
 
+  private static String formatComments(String text) {
+    return Splitter.on('\n').omitEmptyStrings().trimResults().splitToList(text).stream()
+        .map(s -> "# " + s)
+        .collect(Collectors.joining("\n"));
+  }
+
   private RegistryConfig() {}
 }

java/google/registry/config/RegistryConfigSettings.java

@@ -87,6 +87,7 @@ public class RegistryConfigSettings {
     public String tmchMarksDbUrl;
     public String checkApiServletClientId;
     public String registryAdminClientId;
+    public String premiumTermsExportDisclaimer;
     public String reservedTermsExportDisclaimer;
     public String whoisDisclaimer;
   }

java/google/registry/config/files/default-config.yaml

@@ -77,6 +77,12 @@ registryPolicy:
   # registrar
   registryAdminClientId: TheRegistrar
 
+  # Disclaimer at the top of the exported premium terms list.
+  premiumTermsExportDisclaimer: |
+    This list contains domains for the TLD offered at a premium price. This
+    list is subject to change. The most up-to-date source is always the
+    registry itself, by sending domain check EPP commands.
+
   # Disclaimer at the top of the exported reserved terms list.
   reservedTermsExportDisclaimer: |
     This list contains reserved terms for the TLD. Other terms may be reserved

java/google/registry/env/alpha/default/WEB-INF/cron.xml

@@ -178,6 +178,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
+    <description>
+      Premium terms export to Google Drive job for creating once-daily exports.
+    </description>
+    <schedule>every day 05:00</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
     <description>

java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -280,6 +280,12 @@
     <url-pattern>/_dr/task/syncRegistrarsSheet</url-pattern>
   </servlet-mapping>
 
+  <!-- Exports TLD premium terms. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/exportPremiumTerms</url-pattern>
+  </servlet-mapping>
+
   <!-- Exports TLD reserved terms. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

java/google/registry/export/ExportPremiumTermsAction.java

@@ -0,0 +1,150 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static google.registry.model.registry.label.PremiumListUtils.loadPremiumListEntries;
+import static google.registry.request.Action.Method.POST;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.registry.Registry;
+import google.registry.model.registry.label.PremiumList;
+import google.registry.request.Action;
+import google.registry.request.Parameter;
+import google.registry.request.RequestParameters;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.storage.drive.DriveConnection;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.SortedSet;
+import javax.inject.Inject;
+
+/** Action that exports the premium terms list for a TLD to Google Drive. */
+@Action(path = "/_dr/task/exportPremiumTerms", method = POST, auth = Auth.AUTH_INTERNAL_ONLY)
+public class ExportPremiumTermsAction implements Runnable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  static final MediaType EXPORT_MIME_TYPE = MediaType.PLAIN_TEXT_UTF_8;
+  static final String PREMIUM_TERMS_FILENAME = "CONFIDENTIAL_premium_terms.txt";
+
+  @Inject DriveConnection driveConnection;
+
+  @Inject
+  @Config("premiumTermsExportDisclaimer")
+  String exportDisclaimer;
+
+  @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
+  @Inject Response response;
+
+  @Inject
+  ExportPremiumTermsAction() {}
+
+  /**
+   * Exports the premium terms for the TLD specified via the "tld" param to a file in the Google
+   * Drive folder configured for that TLD.
+   *
+   * <p>The export file is named "CONFIDENTIAL_premium_terms.txt" and is encoded in UTF-8. It begins
+   * with the disclaimer text that is immediately followed by premium terms, each occupying a line.
+   * The file ends with a trailing newline.
+   *
+   * <p>Each term is formatted as "term,price", where price is the ISO-4217 three-letter currency
+   * code followed by a space and then the numeric amount. For example:
+   *
+   * <pre>
+   * bank,USD 1599.00
+   * </pre>
+   *
+   * <p>This servlet prints the ID of the file in GoogleDrive that was created/updated.
+   */
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+    try {
+      Registry registry = Registry.get(tld);
+      String resultMsg = checkConfig(registry).orElseGet(() -> exportPremiumTerms(registry));
+      response.setStatus(SC_OK);
+      response.setPayload(resultMsg);
+    } catch (Throwable e) {
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(e.getMessage());
+      throw new RuntimeException(
+          String.format("Exception occurred while exporting premium terms for TLD %s.", tld), e);
+    }
+  }
+
+  /**
+   * Checks if {@code registry} is properly configured to export premium terms.
+   *
+   * @return {@link Optional#empty()} if {@code registry} export may proceed. Otherwise returns an
+   *     error message
+   */
+  private Optional<String> checkConfig(Registry registry) {
+    if (isNullOrEmpty(registry.getDriveFolderId())) {
+      logger.atInfo().log(
+          "Skipping premium terms export for TLD %s because Drive folder isn't specified", tld);
+      return Optional.of("Skipping export because no Drive folder is associated with this TLD");
+    }
+    if (registry.getPremiumList() == null) {
+      logger.atInfo().log("No premium terms to export for TLD %s", tld);
+      return Optional.of("No premium lists configured");
+    }
+    return Optional.empty();
+  }
+
+  private String exportPremiumTerms(Registry registry) {
+    try {
+      String fileId =
+          driveConnection.createOrUpdateFile(
+              PREMIUM_TERMS_FILENAME,
+              EXPORT_MIME_TYPE,
+              registry.getDriveFolderId(),
+              getFormattedPremiumTerms(registry).getBytes(UTF_8));
+      logger.atInfo().log(
+          "Exporting premium terms succeeded for TLD %s, file ID is: %s", tld, fileId);
+      return fileId;
+    } catch (IOException e) {
+      throw new RuntimeException("Error exporting premium terms file to Drive.", e);
+    }
+  }
+
+  private String getFormattedPremiumTerms(Registry registry) {
+    Optional<PremiumList> premiumList = PremiumList.getCached(registry.getPremiumList().getName());
+    checkState(premiumList.isPresent(), "Could not load premium list for " + tld);
+    SortedSet<String> premiumTerms =
+        Streams.stream(loadPremiumListEntries(premiumList.get()))
+            .map(entry -> Joiner.on(",").join(entry.getLabel(), entry.getValue()))
+            .collect(ImmutableSortedSet.toImmutableSortedSet(String::compareTo));
+
+    return Joiner.on("\n")
+        .appendTo(
+            new StringBuilder(),
+            Iterables.concat(ImmutableList.of(exportDisclaimer.trim()), premiumTerms))
+        .append("\n")
+        .toString();
+  }
+}

java/google/registry/model/registry/label/PremiumListUtils.java

@@ -213,6 +213,15 @@ public final class PremiumListUtils {
     ofy().transactNew(() -> ofy().delete().key(premiumList.getRevisionKey()));
   }
 
+  /**
+   * Returns all {@link PremiumListEntry PremiumListEntries} in the given {@code premiumList}.
+   *
+   * <p>This is an expensive operation and should only be used when the entire list is required.
+   */
+  public static Iterable<PremiumListEntry> loadPremiumListEntries(PremiumList premiumList) {
+    return ofy().load().type(PremiumListEntry.class).ancestor(premiumList.revisionKey).iterable();
+  }
+
   /** Returns whether a PremiumList of the given name exists, bypassing the cache. */
   public static boolean doesPremiumListExist(String name) {
     return ofy().load().key(Key.create(getCrossTldKey(), PremiumList.class, name)).now() != null;

java/google/registry/module/backend/BackendRequestComponent.java

@@ -44,6 +44,7 @@ import google.registry.dns.writer.dnsupdate.DnsUpdateWriterModule;
 import google.registry.export.BigqueryPollJobAction;
 import google.registry.export.CheckSnapshotAction;
 import google.registry.export.ExportDomainListsAction;
+import google.registry.export.ExportPremiumTermsAction;
 import google.registry.export.ExportRequestModule;
 import google.registry.export.ExportReservedTermsAction;
 import google.registry.export.ExportSnapshotAction;
@@ -129,6 +130,7 @@ interface BackendRequestComponent {
   ExpandRecurringBillingEventsAction expandRecurringBillingEventsAction();
   ExportCommitLogDiffAction exportCommitLogDiffAction();
   ExportDomainListsAction exportDomainListsAction();
+  ExportPremiumTermsAction exportPremiumTermsAction();
   ExportReservedTermsAction exportReservedTermsAction();
   ExportSnapshotAction exportSnapshotAction();
   GenerateInvoicesAction generateInvoicesAction();

javatests/google/registry/export/ExportPremiumTermsActionTest.java

@@ -0,0 +1,195 @@
+// Copyright 2018 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.export;
+
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static google.registry.export.ExportPremiumTermsAction.EXPORT_MIME_TYPE;
+import static google.registry.export.ExportPremiumTermsAction.PREMIUM_TERMS_FILENAME;
+import static google.registry.model.registry.label.PremiumListUtils.deletePremiumList;
+import static google.registry.model.registry.label.PremiumListUtils.savePremiumListAndEntries;
+import static google.registry.testing.DatastoreHelper.createTld;
+import static google.registry.testing.DatastoreHelper.deleteTld;
+import static google.registry.testing.DatastoreHelper.persistResource;
+import static google.registry.testing.JUnitBackports.assertThrows;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.net.MediaType;
+import google.registry.model.registry.Registry;
+import google.registry.model.registry.label.PremiumList;
+import google.registry.request.Response;
+import google.registry.storage.drive.DriveConnection;
+import google.registry.testing.AppEngineRule;
+import java.io.IOException;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Matchers;
+
+@RunWith(JUnit4.class)
+public class ExportPremiumTermsActionTest {
+
+  private static final String DISCLAIMER_WITH_NEWLINE = "# Premium Terms Export Disclaimer\n";
+  private static final ImmutableList<String> PREMIUM_NAMES =
+      ImmutableList.of("2048,USD 549", "0,USD 549");
+  private static final String EXPECTED_FILE_CONTENT =
+      DISCLAIMER_WITH_NEWLINE + "0,USD 549.00\n" + "2048,USD 549.00\n";
+
+  @Rule public final AppEngineRule appEngine = AppEngineRule.builder().withDatastore().build();
+
+  private final DriveConnection driveConnection = mock(DriveConnection.class);
+  private final Response response = mock(Response.class);
+
+  private void runAction(String tld) {
+    ExportPremiumTermsAction action = new ExportPremiumTermsAction();
+    action.response = response;
+    action.driveConnection = driveConnection;
+    action.exportDisclaimer = DISCLAIMER_WITH_NEWLINE;
+    action.tld = tld;
+    action.run();
+  }
+
+  @Before
+  public void setup() throws Exception {
+    createTld("tld");
+    PremiumList pl = new PremiumList.Builder().setName("pl-name").build();
+    savePremiumListAndEntries(pl, PREMIUM_NAMES);
+    persistResource(
+        Registry.get("tld").asBuilder().setPremiumList(pl).setDriveFolderId("folder_id").build());
+    when(driveConnection.createOrUpdateFile(
+            anyString(), any(MediaType.class), eq("folder_id"), any(byte[].class)))
+        .thenReturn("file_id");
+    when(driveConnection.createOrUpdateFile(
+            anyString(), any(MediaType.class), eq("bad_folder_id"), any(byte[].class)))
+        .thenThrow(new IOException());
+  }
+
+  @Test
+  public void test_exportPremiumTerms_success() throws IOException {
+    runAction("tld");
+
+    verify(driveConnection)
+        .createOrUpdateFile(
+            PREMIUM_TERMS_FILENAME,
+            EXPORT_MIME_TYPE,
+            "folder_id",
+            EXPECTED_FILE_CONTENT.getBytes(UTF_8));
+    verifyNoMoreInteractions(driveConnection);
+
+    verify(response).setStatus(SC_OK);
+    verify(response).setPayload("file_id");
+    verify(response).setContentType(PLAIN_TEXT_UTF_8);
+    verifyNoMoreInteractions(response);
+  }
+
+  @Test
+  public void test_exportPremiumTerms_success_emptyPremiumList() throws IOException {
+    PremiumList pl = new PremiumList.Builder().setName("pl-name").build();
+    savePremiumListAndEntries(pl, ImmutableList.of());
+    runAction("tld");
+
+    verify(driveConnection)
+        .createOrUpdateFile(
+            PREMIUM_TERMS_FILENAME,
+            EXPORT_MIME_TYPE,
+            "folder_id",
+            DISCLAIMER_WITH_NEWLINE.getBytes(UTF_8));
+    verifyNoMoreInteractions(driveConnection);
+
+    verify(response).setStatus(SC_OK);
+    verify(response).setPayload("file_id");
+    verify(response).setContentType(PLAIN_TEXT_UTF_8);
+    verifyNoMoreInteractions(response);
+  }
+
+  @Test
+  public void test_exportPremiumTerms_doNothing_listNotConfigured() {
+    persistResource(Registry.get("tld").asBuilder().setPremiumList(null).build());
+    runAction("tld");
+
+    verifyZeroInteractions(driveConnection);
+    verify(response).setStatus(SC_OK);
+    verify(response).setPayload("No premium lists configured");
+    verify(response).setContentType(PLAIN_TEXT_UTF_8);
+    verifyNoMoreInteractions(response);
+  }
+
+  @Test
+  public void testExportPremiumTerms_doNothing_driveIdNotConfiguredInTld() {
+    persistResource(Registry.get("tld").asBuilder().setDriveFolderId(null).build());
+    runAction("tld");
+
+    verifyZeroInteractions(driveConnection);
+    verify(response).setStatus(SC_OK);
+    verify(response)
+        .setPayload("Skipping export because no Drive folder is associated with this TLD");
+    verify(response).setContentType(PLAIN_TEXT_UTF_8);
+    verifyNoMoreInteractions(response);
+  }
+
+  @Test
+  public void test_exportPremiumTerms_failure_noSuchTld() {
+    deleteTld("tld");
+    assertThrows(RuntimeException.class, () -> runAction("tld"));
+
+    verifyZeroInteractions(driveConnection);
+    verify(response).setStatus(SC_INTERNAL_SERVER_ERROR);
+    verify(response).setPayload(anyString());
+    verify(response).setContentType(PLAIN_TEXT_UTF_8);
+    verifyNoMoreInteractions(response);
+  }
+
+  @Test
+  public void test_exportPremiumTerms_failure_noPremiumList() {
+    deletePremiumList(new PremiumList.Builder().setName("pl-name").build());
+    assertThrows(RuntimeException.class, () -> runAction("tld"));
+
+    verifyZeroInteractions(driveConnection);
+    verify(response).setStatus(SC_INTERNAL_SERVER_ERROR);
+    verify(response).setPayload("Could not load premium list for " + "tld");
+    verify(response).setContentType(PLAIN_TEXT_UTF_8);
+    verifyNoMoreInteractions(response);
+  }
+
+  @Test
+  public void testExportPremiumTerms_failure_driveIdThrowsException() throws IOException {
+    persistResource(Registry.get("tld").asBuilder().setDriveFolderId("bad_folder_id").build());
+    assertThrows(RuntimeException.class, () -> runAction("tld"));
+
+    verify(driveConnection)
+        .createOrUpdateFile(
+            PREMIUM_TERMS_FILENAME,
+            EXPORT_MIME_TYPE,
+            "bad_folder_id",
+            EXPECTED_FILE_CONTENT.getBytes(UTF_8));
+    verifyNoMoreInteractions(driveConnection);
+    verify(response).setStatus(SC_INTERNAL_SERVER_ERROR);
+    verify(response).setPayload(Matchers.contains("Error exporting premium terms file to Drive."));
+    verify(response).setContentType(PLAIN_TEXT_UTF_8);
+    verifyNoMoreInteractions(response);
+  }
+}

javatests/google/registry/module/backend/testdata/backend_routing.txt

@@ -14,6 +14,7 @@ PATH                                   CLASS                              METHOD
 /_dr/task/expandRecurringBillingEvents ExpandRecurringBillingEventsAction GET      n  INTERNAL     APP IGNORED
 /_dr/task/exportCommitLogDiff          ExportCommitLogDiffAction          POST     y  INTERNAL     APP IGNORED
 /_dr/task/exportDomainLists            ExportDomainListsAction            POST     n  INTERNAL     APP IGNORED
+/_dr/task/exportPremiumTerms           ExportPremiumTermsAction           POST     n  INTERNAL     APP IGNORED
 /_dr/task/exportReservedTerms          ExportReservedTermsAction          POST     n  INTERNAL     APP IGNORED
 /_dr/task/exportSnapshot               ExportSnapshotAction               POST     y  INTERNAL     APP IGNORED
 /_dr/task/generateInvoices             GenerateInvoicesAction             POST     n  INTERNAL     APP IGNORED