mirror of
https://github.com/google/nomulus.git
synced 2025-05-04 05:57:51 +02:00
552ab12314
This makes a few cosmetic changes that prepares the pipeline for production. Namely: - Converts file names to include the input yearMonth, mostly mirroring the original invoicing pipeline. - Factors out the yearMonth logic from the reporting module to the more common backend module. We will likely use the default yearMonth logic in other backend tasks (such as spec11 reporting). - Adds the "withTemplateCompatability" flag to the Bigquery read, which allows multiple uses of the same template. - Adds the 'billing' task queue, which retries up to 5 times every 3 minutes, which is about the rate we desire for checking if the pipeline is complete. - Adds a shell 'invoicing upload' class, which tests the retry semantics we want for post-generation work (e-mailing the invoice to crr-tech, and publishing detail reports) While this cl may look big, it's mostly just a refactor and setting up boilerplate needed to frame the upload logic. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=179849586
406 lines
14 KiB
XML
406 lines
14 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5"
|
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee
|
|
http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
|
|
<!-- Servlets -->
|
|
|
|
<!-- Servlet for injected backends actions -->
|
|
<servlet>
|
|
<display-name>BackendServlet</display-name>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<servlet-class>google.registry.module.backend.BackendServlet</servlet-class>
|
|
<load-on-startup>1</load-on-startup>
|
|
</servlet>
|
|
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/metrics</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- RDE -->
|
|
|
|
<!--
|
|
Responsible for scanning the database to create a full deposit for a single TLD
|
|
and streaming it to cloud storage. Requests are sent here by App Engine after
|
|
`RdeCreateCronServlet` enqueues a task specifying a URL that points to this servlet.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/rdeStaging</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!--
|
|
Once `rdeCreateFullCron` finishes writing a deposit file to cloud storage, it'll
|
|
launch this task with the cloud filename so it can be uploaded to Iron Mountain
|
|
via SFTP. The file is deleted once the upload completes. This should be run via
|
|
`rde-upload-backend`.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/rdeUpload</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Sends an XML RDE report to ICANN's HTTP server after rdeUploadTask finishes. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/rdeReport</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!--
|
|
Bulk Registration Data Access. This task creates a thin escrow deposit
|
|
and saves it to cloud storage, where a separate script owned by the SREs
|
|
uploads it to ICANN.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/brdaCopy</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Billing -->
|
|
|
|
<!--
|
|
Generates the invoice CSV for the month, which we send to billing to charge
|
|
registrars for their registrations.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/generateInvoices</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!--
|
|
Emails the month's invoice CSV to the internal billing team, and publishes
|
|
the detail reports to the individual registrars' drive accounts.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/publishInvoices</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- ICANN Monthly Reporting -->
|
|
|
|
<!--
|
|
Monthly ICANN transaction and activity reports. This task generates report
|
|
files (in CSV format) and stores them in GCS under
|
|
gs://domain-registry-reporting/icann/monthly/YYYY-MM
|
|
by default.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/icannReportingStaging</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!--
|
|
Monthly ICANN transaction and activity reports. This task uploads the generated
|
|
report files (in CSV format) via an HTTP PUT to ICANN's endpoint.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/icannReportingUpload</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Trademark Clearinghouse -->
|
|
|
|
<!-- Downloads TMCH DNL data from MarksDB. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/tmchDnl</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Downloads TMCH SMDRL data from MarksDB. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/tmchSmdrl</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Downloads TMCH CRL data from MarksDB. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/tmchCrl</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Reads the LORDN queues and uploads CSV data for sunrise and claims marks to MarksDB. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/nordnUpload</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Verifies upload of LORDN data to MarksDB. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/nordnVerify</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Reads the DNS push and pull queues and kick off the appropriate tasks to update zone. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/cron/readDnsQueue</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Publishes DNS updates. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/publishDnsUpdates</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Manually refreshes DNS information. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/dnsRefresh</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Verifies integrity of database invariants. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/verifyEntityIntegrity</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Exports a Datastore backup snapshot to GCS. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/exportSnapshot</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Checks the completion of a Datastore backup snapshot. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/checkSnapshot</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Loads a Datastore backup snapshot into BigQuery. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/loadSnapshot</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Updates a view to point at a certain snapshot in BigQuery. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/updateSnapshotView</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Polls state of jobs in Bigquery -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/pollBigqueryJob</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Cleans up old mapreduce entities. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/mapreduceEntityCleanup</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Fans out a cron task over an adjustable range of TLDs. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/cron/fanout</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Backups. -->
|
|
|
|
<!-- Fans out a cron task over all commit log buckets. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/cron/commitLogFanout</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Deletes old commit logs from Datastore. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/deleteOldCommitLogs</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Checkpoints commit logs. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/cron/commitLogCheckpoint</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Exports commit log diffs. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/exportCommitLogDiff</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Deletes commit logs. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/killCommitLogs</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- MapReduce servlet. -->
|
|
<servlet>
|
|
<servlet-name>mapreduce</servlet-name>
|
|
<servlet-class>com.google.appengine.tools.mapreduce.MapReduceServlet</servlet-class>
|
|
</servlet>
|
|
<servlet-mapping>
|
|
<servlet-name>mapreduce</servlet-name>
|
|
<url-pattern>/_dr/mapreduce/*</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Pipeline GUI servlets. -->
|
|
<servlet>
|
|
<servlet-name>pipeline</servlet-name>
|
|
<servlet-class>com.google.appengine.tools.pipeline.impl.servlets.PipelineServlet</servlet-class>
|
|
</servlet>
|
|
<servlet-mapping>
|
|
<servlet-name>pipeline</servlet-name>
|
|
<url-pattern>/_ah/pipeline/*</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Syncs registrars to the registrar spreadsheet. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/syncRegistrarsSheet</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Exports TLD reserved terms. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/exportReservedTerms</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Syncs RegistrarContact changes to Google Groups. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/syncGroupMembers</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/exportDomainLists</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Mapreduce to delete all prober data. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/deleteProberData</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Mapreduce to re-save all EppResources. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/resaveAllEppResources</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!--
|
|
Deletes contacts and hosts enqueued for asynchronous deletion if they are
|
|
not referenced by any domain.
|
|
-->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/deleteContactsAndHosts</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Enqueues DNS update tasks following a host rename. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/dnsRefreshForHostRename</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Enqueues DNS update tasks following a host rename. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/refreshDnsOnHostRename</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Mapreduce to expand recurring billing events into OneTimes. -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/expandRecurringBillingEvents</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Mapreduce to import contacts from escrow file -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/importRdeContacts</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Mapreduce to import hosts from escrow file -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/importRdeHosts</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Mapreduce to import domains from escrow file -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/importRdeDomains</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Mapreduce to link hosts from escrow file to superordinate domains -->
|
|
<servlet-mapping>
|
|
<servlet-name>backend-servlet</servlet-name>
|
|
<url-pattern>/_dr/task/linkRdeHosts</url-pattern>
|
|
</servlet-mapping>
|
|
|
|
<!-- Security config -->
|
|
<security-constraint>
|
|
<web-resource-collection>
|
|
<web-resource-name>Internal</web-resource-name>
|
|
<description>
|
|
Admin-only internal section. Requests for paths covered by the URL patterns below will be
|
|
checked for a logged-in user account that's allowed to access the AppEngine admin console
|
|
(NOTE: this includes Editor/Viewer permissions in addition to Owner and the new IAM
|
|
App Engine Admin role. See https://cloud.google.com/appengine/docs/java/access-control
|
|
specifically the "Access handlers that have a login:admin restriction" line.)
|
|
|
|
TODO(b/28219927): lift some of these restrictions so that we can allow OAuth authentication
|
|
for endpoints that need to be accessed by open-source automated processes.
|
|
</description>
|
|
|
|
<!-- Internal AppEngine endpoints. The '_ah' is short for app hosting. -->
|
|
<url-pattern>/_ah/*</url-pattern>
|
|
|
|
<!-- Registrar console (should not be available on non-default module). -->
|
|
<url-pattern>/registrar*</url-pattern>
|
|
|
|
<!-- Verbatim JavaScript sources (only visible to admins for debugging). -->
|
|
<url-pattern>/assets/sources/*</url-pattern>
|
|
|
|
</web-resource-collection>
|
|
<auth-constraint>
|
|
<role-name>admin</role-name>
|
|
</auth-constraint>
|
|
|
|
<!-- Repeated here since catch-all rule below is not inherited. -->
|
|
<user-data-constraint>
|
|
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
|
|
</user-data-constraint>
|
|
</security-constraint>
|
|
|
|
<!-- Require TLS on all requests. -->
|
|
<security-constraint>
|
|
<web-resource-collection>
|
|
<web-resource-name>Secure</web-resource-name>
|
|
<description>
|
|
Require encryption for all paths. http URLs will be redirected to https.
|
|
</description>
|
|
<url-pattern>/*</url-pattern>
|
|
</web-resource-collection>
|
|
<user-data-constraint>
|
|
<transport-guarantee>CONFIDENTIAL</transport-guarantee>
|
|
</user-data-constraint>
|
|
</security-constraint>
|
|
|
|
<!-- See: https://code.google.com/p/objectify-appengine/wiki/Setup -->
|
|
<filter>
|
|
<filter-name>ObjectifyFilter</filter-name>
|
|
<filter-class>com.googlecode.objectify.ObjectifyFilter</filter-class>
|
|
</filter>
|
|
<filter-mapping>
|
|
<filter-name>ObjectifyFilter</filter-name>
|
|
<url-pattern>/*</url-pattern>
|
|
</filter-mapping>
|
|
|
|
<!-- Register types with Objectify. -->
|
|
<filter>
|
|
<filter-name>OfyFilter</filter-name>
|
|
<filter-class>google.registry.model.ofy.OfyFilter</filter-class>
|
|
</filter>
|
|
<filter-mapping>
|
|
<filter-name>OfyFilter</filter-name>
|
|
<url-pattern>/*</url-pattern>
|
|
</filter-mapping>
|
|
</web-app>
|