// Copyright 2017 The Nomulus Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package google.registry.tmch; import static com.google.appengine.api.taskqueue.QueueFactory.getQueue; import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl; import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; import static com.google.appengine.api.urlfetch.HTTPMethod.POST; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.net.HttpHeaders.LOCATION; import static com.google.common.net.MediaType.CSV_UTF_8; import static google.registry.tmch.LordnTaskUtils.COLUMNS_CLAIMS; import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE; import static google.registry.util.UrlFetchUtils.getHeaderFirst; import static google.registry.util.UrlFetchUtils.setPayloadMultipart; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED; import com.google.appengine.api.taskqueue.LeaseOptions; import com.google.appengine.api.taskqueue.Queue; import com.google.appengine.api.taskqueue.TaskHandle; import com.google.appengine.api.taskqueue.TaskOptions; import com.google.appengine.api.taskqueue.TransientFailureException; import com.google.appengine.api.urlfetch.HTTPRequest; import com.google.appengine.api.urlfetch.HTTPResponse; import com.google.appengine.api.urlfetch.URLFetchService; import com.google.apphosting.api.DeadlineExceededException; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import google.registry.config.RegistryConfig.Config; import google.registry.request.Action; import google.registry.request.Parameter; import google.registry.request.RequestParameters; import google.registry.request.auth.Auth; import google.registry.util.Clock; import google.registry.util.Retrier; import google.registry.util.TaskQueueUtils; import google.registry.util.UrlFetchException; import java.io.IOException; import java.net.URL; import java.util.List; import java.util.Optional; import java.util.Random; import java.util.concurrent.TimeUnit; import javax.inject.Inject; import org.joda.time.DateTime; import org.joda.time.Duration; /** * Action that reads the NORDN pull queues, uploads claims and sunrise marks data to TMCH, and * enqueues subsequent upload verification tasks. A unique actionLogId is generated and passed along * to the verify action so that connected verify tasks can be identified by looking at logs. * * @see NordnVerifyAction */ @Action( path = NordnUploadAction.PATH, method = Action.Method.POST, automaticallyPrintOk = true, auth = Auth.AUTH_INTERNAL_ONLY) public final class NordnUploadAction implements Runnable { static final String PATH = "/_dr/task/nordnUpload"; static final String LORDN_PHASE_PARAM = "lordn-phase"; private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final Duration LEASE_PERIOD = Duration.standardHours(1); /** * A unique (enough) id that is outputted in log lines to make it clear which log lines are * associated with a given invocation of the NordnUploadAction in the event that multiple * instances execute simultaneously. */ private final String actionLogId = String.valueOf(1000000000 + new Random().nextInt(1000000000)); @Inject Clock clock; @Inject Retrier retrier; @Inject @Config("insecureRandom") Random random; @Inject LordnRequestInitializer lordnRequestInitializer; @Inject URLFetchService fetchService; @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl; @Inject @Parameter(LORDN_PHASE_PARAM) String phase; @Inject @Parameter(RequestParameters.PARAM_TLD) String tld; @Inject TaskQueueUtils taskQueueUtils; @Inject NordnUploadAction() {} /** * These LORDN parameter names correspond to the relative paths in LORDN URLs and cannot be * changed on our end. */ private static final String PARAM_LORDN_PHASE_SUNRISE = "sunrise"; private static final String PARAM_LORDN_PHASE_CLAIMS = "claims"; /** How long to wait before attempting to verify an upload by fetching the log. */ private static final Duration VERIFY_DELAY = Duration.standardMinutes(30); @Override public void run() { try { processLordnTasks(); } catch (IOException e) { throw new RuntimeException(e); } } /** * Converts a list of queue tasks, each containing a row of CSV data, into a single newline- * delimited String. */ static String convertTasksToCsv(List tasks, DateTime now, String columns) { String header = String.format("1,%s,%d\n%s\n", now, tasks.size(), columns); StringBuilder csv = new StringBuilder(header); for (TaskHandle task : checkNotNull(tasks)) { String payload = new String(task.getPayload(), UTF_8); if (!Strings.isNullOrEmpty(payload)) { csv.append(payload).append("\n"); } } return csv.toString(); } /** Leases and returns all tasks from the queue with the specified tag tld, in batches. */ List loadAllTasks(Queue queue, String tld) { ImmutableList.Builder allTasks = new ImmutableList.Builder<>(); while (true) { List tasks = retrier.callWithRetry( () -> queue.leaseTasks( LeaseOptions.Builder.withTag(tld) .leasePeriod(LEASE_PERIOD.getMillis(), TimeUnit.MILLISECONDS) .countLimit(TaskQueueUtils.getBatchSize())), TransientFailureException.class, DeadlineExceededException.class); if (tasks.isEmpty()) { return allTasks.build(); } allTasks.addAll(tasks); } } private void processLordnTasks() throws IOException { checkArgument(phase.equals(PARAM_LORDN_PHASE_SUNRISE) || phase.equals(PARAM_LORDN_PHASE_CLAIMS), "Invalid phase specified to Nordn servlet: %s.", phase); DateTime now = clock.nowUtc(); Queue queue = getQueue( phase.equals(PARAM_LORDN_PHASE_SUNRISE) ? LordnTaskUtils.QUEUE_SUNRISE : LordnTaskUtils.QUEUE_CLAIMS); String columns = phase.equals(PARAM_LORDN_PHASE_SUNRISE) ? COLUMNS_SUNRISE : COLUMNS_CLAIMS; List tasks = loadAllTasks(queue, tld); if (!tasks.isEmpty()) { String csvData = convertTasksToCsv(tasks, now, columns); uploadCsvToLordn(String.format("/LORDN/%s/%s", tld, phase), csvData); taskQueueUtils.deleteTasks(queue, tasks); } } /** * Upload LORDN file to MarksDB. * *

Idempotency: If the exact same LORDN report is uploaded twice, the MarksDB server will * return the same confirmation number. * * @see * TMCH functional specifications - LORDN File */ private void uploadCsvToLordn(String urlPath, String csvData) throws IOException { String url = tmchMarksdbUrl + urlPath; logger.atInfo().log( "LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData); HTTPRequest req = new HTTPRequest(new URL(url), POST, validateCertificate().setDeadline(60d)); lordnRequestInitializer.initialize(req, tld); setPayloadMultipart(req, "file", "claims.csv", CSV_UTF_8, csvData, random); HTTPResponse rsp = fetchService.fetch(req); logger.atInfo().log( "LORDN upload task %s response: HTTP response code %d, response data: %s", actionLogId, rsp.getResponseCode(), rsp.getContent()); if (rsp.getResponseCode() != SC_ACCEPTED) { throw new UrlFetchException( String.format( "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", actionLogId), req, rsp); } Optional location = getHeaderFirst(rsp, LOCATION); if (!location.isPresent()) { throw new UrlFetchException( String.format( "LORDN upload task %s error: MarksDB failed to provide a Location header", actionLogId), req, rsp); } getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location.get()))); } private TaskOptions makeVerifyTask(URL url) { // The actionLogId is used to uniquely associate the verify task back to the upload task. return withUrl(NordnVerifyAction.PATH) .header(NordnVerifyAction.URL_HEADER, url.toString()) .header(NordnVerifyAction.HEADER_ACTION_LOG_ID, actionLogId) .param(RequestParameters.PARAM_TLD, tld) .countdownMillis(VERIFY_DELAY.getMillis()); } }