zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-04-29 19:47:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

Move App Engine cron jobs to cloud scheduler (#1939)

Browse source
This commit is contained in:
Pavlo Tkach 2023-03-01 13:40:56 -05:00 • committed by GitHub
parent 7f3a43f70d
commit f615e88ff6
21 changed files with 1272 additions and 898 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
core/src/main/java/google/registry/config/RegistryConfig.java
View file

@ -106,6 +106,12 @@ public final class RegistryConfig {
return config.gcpProject.projectId;
}
@Provides
@Config("cloudSchedulerServiceAccountEmail")
public static String provideCloudSchedulerServiceAccountEmail(RegistryConfigSettings config) {
return config.gcpProject.cloudSchedulerServiceAccountEmail;
}
@Provides
@Config("projectIdNumber")
public static long provideProjectIdNumber(RegistryConfigSettings config) {

1
core/src/main/java/google/registry/config/RegistryConfigSettings.java
View file

@ -54,6 +54,7 @@ public class RegistryConfigSettings {
public String backendServiceUrl;
public String toolsServiceUrl;
public String pubapiServiceUrl;
public String cloudSchedulerServiceAccountEmail;
}
/** Configuration options for OAuth settings for authenticating users. */

2
core/src/main/java/google/registry/config/files/default-config.yaml
View file

@ -22,6 +22,8 @@ gcpProject:
backendServiceUrl: https://localhost
toolsServiceUrl: https://localhost
pubapiServiceUrl: https://localhost
# Service account used by Cloud Scheduler to send authenticated requests.
cloudSchedulerServiceAccountEmail: cloud-scheduler-email@email.com
gSuite:
# Publicly accessible domain name of the running G Suite instance.

140
core/src/main/java/google/registry/env/alpha/default/WEB-INF/cloud-scheduler-tasks.xml vendored Normal file
View file

@ -0,0 +1,140 @@
<?xml version="1.0" encoding="UTF-8"?>
<taskentries>
<task>
<url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<schedule>7 0 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<name>rdeUpload</name>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>0 */4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<name>tmchDnl</name>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<name>tmchSmdrl</name>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>15 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<name>tmchCrl</name>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<name>syncGroupMembers</name>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<name>syncRegistrarsSheet</name>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<name>resaveAllEppResourcesPipeline</name>
<description>
This job resaves all our resources, projected in time to "now".
</description>
<!--Deviation from cron tasks schedule: 1st monday of month 09:00 is replaced
with 1st of the month 09:00 -->
<schedule>0 9 1 * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents?advanceCursor]]></url>
<name>expandRecurringBillingEvents</name>
<description>
This job runs an action that creates synthetic OneTime billing events from Recurring billing
events. Events are created for all instances of Recurring billing events that should exist
between the RECURRING_BILLING cursor's time and the execution time of the action.
</description>
<schedule>0 3 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<name>deleteExpiredDomains</name>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>7 3 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<name>deleteProberData</name>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>0 14 * * 1</schedule>
</task>
<!-- TODO: Add borgmon job to check that these files are created and updated successfully. -->
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<name>exportReservedTerms</name>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>30 5 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<name>exportPremiumTerms</name>
<description>
Premium terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>0 5 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<name>readDnsQueue</name>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>*/1 * * * *</schedule>
</task>
</taskentries>

147
core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml vendored
View file

@ -1,150 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--TODO: ptkach - Remove this file after cloud scheduler is fully up and running -->
<cronentries>
<!--
/cron/fanout params:
queue=<QUEUE_NAME>
endpoint=<ENDPOINT_NAME> // URL Path of servlet, which may contain placeholders:
runInEmpty // Run once, with no tld parameter
forEachRealTld // Run for tlds with getTldType() == TldType.REAL
forEachTestTld // Run for tlds with getTldType() == TldType.TEST
exclude=TLD1[,TLD2] // exclude something otherwise included
-->
<cron>
<url>/_dr/task/rdeStaging</url>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<schedule>every day 00:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>every 4 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>every 12 hours from 00:15 to 12:15</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description>
This job resaves all our resources, projected in time to "now".
</description>
<schedule>1st monday of month 09:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents?advanceCursor]]></url>
<description>
This job runs an action that creates synthetic OneTime billing events from Recurring billing
events. Events are created for all instances of Recurring billing events that should exist
between the RECURRING_BILLING cursor's time and the execution time of the action.
</description>
<schedule>every day 03:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>every day 03:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>every monday 14:00</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<!-- TODO: Add borgmon job to check that these files are created and updated successfully. -->
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>every day 05:30</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<description>
Premium terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>every day 05:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>every 1 minutes synchronized</schedule>
<target>backend</target>
</cron>
</cronentries>

163
core/src/main/java/google/registry/env/crash/default/WEB-INF/cloud-scheduler-tasks.xml vendored Normal file
View file

@ -0,0 +1,163 @@
<?xml version="1.0" encoding="UTF-8"?>
<taskentries>
<!--
/cron/fanout params:
queue=<QUEUE_NAME>
endpoint=<ENDPOINT_NAME> // URL Path of servlet, which may contain placeholders:
runInEmpty // Run once, with no tld parameter
forEachRealTld // Run for tlds with getTldType() == TldType.REAL
forEachTestTld // Run for tlds with getTldType() == TldType.TEST
exclude=TLD1[,TLD2] // exclude something otherwise included
-->
<task>
<url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<!--
This only needs to run once per day, but we launch additional jobs in case the
cursor is lagging behind, so it'll catch up to the current date as quickly as
possible. The only job that'll run under normal circumstances is the one that's
close to midnight, since if the cursor is up-to-date, the task is a no-op.
We want it to be close to midnight because that reduces the chance that the
point-in-time code won't have to go to the extra trouble of fetching old
versions of objects from the database. However, we don't want it to run too
close to midnight, because there's always a chance that a change which was
timestamped before midnight hasn't fully been committed to the database. So
we add a 4+ minute grace period to ensure the transactions cool down, since
our queries are not transactional.
-->
<schedule>7 */4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<name>rdeUpload</name>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>0 */4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=rde-report&endpoint=/_dr/task/rdeReport&forEachRealTld]]></url>
<name>rdeReport</name>
<description>
This job is a no-op unless RdeReportCursor falls behind for some reason.
</description>
<schedule>0 */4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<name>tmchDnl</name>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<name>tmchSmdrl</name>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>15 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<name>tmchCrl</name>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<name>syncGroupMembers</name>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<name>syncRegistrarsSheet</name>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<name>deleteProberData</name>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>0 14 * * 1</schedule>
</task>
<!-- TODO: Add borgmon job to check that these files are created and updated successfully. -->
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<name>exportReservedTerms</name>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>30 5 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<name>exportPremiumTerms</name>
<description>
Exports premium price lists to the Google Drive folders for each TLD once per day.
</description>
<schedule>0 5 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<name>readDnsQueue</name>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>*/1 * * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<name>deleteExpiredDomains</name>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>7 3 * * *</schedule>
</task>
<!--
The next two wipeout jobs are required when crash has production data.
-->
<task>
<url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
<name>wipeOutCloudSql</name>
<description>
This job runs an action that deletes all data in Cloud SQL.
</description>
<schedule>7 3 * * 6</schedule>
</task>
</taskentries>

162
core/src/main/java/google/registry/env/crash/default/WEB-INF/cron.xml vendored
View file

@ -1,165 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--TODO: ptkach - Remove this file after cloud scheduler is fully up and running -->
<cronentries>
<!--
/cron/fanout params:
queue=<QUEUE_NAME>
endpoint=<ENDPOINT_NAME> // URL Path of servlet, which may contain placeholders:
runInEmpty // Run once, with no tld parameter
forEachRealTld // Run for tlds with getTldType() == TldType.REAL
forEachTestTld // Run for tlds with getTldType() == TldType.TEST
exclude=TLD1[,TLD2] // exclude something otherwise included
-->
<cron>
<url>/_dr/task/rdeStaging</url>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<!--
This only needs to run once per day, but we launch additional jobs in case the
cursor is lagging behind, so it'll catch up to the current date as quickly as
possible. The only job that'll run under normal circumstances is the one that's
close to midnight, since if the cursor is up-to-date, the task is a no-op.
We want it to be close to midnight because that reduces the chance that the
point-in-time code won't have to go to the extra trouble of fetching old
versions of objects from the database. However, we don't want it to run too
close to midnight, because there's always a chance that a change which was
timestamped before midnight hasn't fully been committed to the database. So
we add a 4+ minute grace period to ensure the transactions cool down, since
our queries are not transactional.
-->
<schedule>every 4 hours from 00:07 to 20:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>every 4 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=rde-report&endpoint=/_dr/task/rdeReport&forEachRealTld]]></url>
<description>
This job is a no-op unless RdeReportCursor falls behind for some reason.
</description>
<schedule>every 4 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>every 12 hours from 00:15 to 12:15</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>every monday 14:00</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<!-- TODO: Add borgmon job to check that these files are created and updated successfully. -->
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>every day 05:30</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<description>
Exports premium price lists to the Google Drive folders for each TLD once per day.
</description>
<schedule>every day 05:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>every 1 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>every day 03:07</schedule>
<target>backend</target>
</cron>
<!--
The next two wipeout jobs are required when crash has production data.
-->
<cron>
<url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
<description>
This job runs an action that deletes all data in Cloud SQL.
</description>
<schedule>every saturday 03:07</schedule>
<target>backend</target>
</cron>
</cronentries>

288
core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml vendored Normal file
View file

@ -0,0 +1,288 @@
<?xml version="1.0" encoding="UTF-8"?>
<taskentries>
<task>
<url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<!--
This only needs to run once per day, but we launch additional jobs in case the
cursor is lagging behind, so it'll catch up to the current date as quickly as
possible. The only job that'll run under normal circumstances is the one that's
close to midnight, since if the cursor is up-to-date, the task is a no-op.
We want it to be close to midnight because that reduces the chance that the
point-in-time code won't have to go to the extra trouble of fetching old
versions of objects from the database. However, we don't want it to run too
close to midnight, because there's always a chance that a change which was
timestamped before midnight hasn't fully been committed to the database. So
we add a 4+ minute grace period to ensure the transactions cool down, since
our queries are not transactional.
-->
<schedule>7 */8 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<name>rdeUpload</name>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>0 */4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=rde-report&endpoint=/_dr/task/rdeReport&forEachRealTld]]></url>
<name>rdeReport</name>
<description>
This job is a no-op unless RdeReportCursor falls behind for some reason.
</description>
<schedule>0 */4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<name>tmchDnl</name>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>0 0,12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<name>tmchSmdrl</name>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>15 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<name>tmchCrl</name>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<name>syncGroupMembers</name>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<name>syncRegistrarsSheet</name>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<!-- TODO: @ptkach add resaveAllEppResourcesPipelineAction https://cs.opensource.google/nomulus/nomulus/+/master:core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml;l=105 -->
<task>
<url><![CDATA[/_dr/task/updateRegistrarRdapBaseUrls]]></url>
<name>updateRegistrarRdapBaseUrls</name>
<description>
This job reloads all registrar RDAP base URLs from ICANN.
</description>
<schedule>34 2 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>
<name>exportDomainLists</name>
<description>
This job exports lists of all active domain names to Google Drive and Google Cloud Storage.
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents?advanceCursor]]></url>
<name>expandRecurringBillingEvents</name>
<description>
This job runs an action that creates synthetic OneTime billing events from Recurring billing
events. Events are created for all instances of Recurring billing events that should exist
between the RECURRING_BILLING cursor's time and the execution time of the action.
</description>
<schedule>0 3 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<name>deleteExpiredDomains</name>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>7 3 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
<name>sendExpiringCertificateNotificationEmail</name>
<description>
This job runs an action that sends emails to partners if their certificates are expiring soon.
</description>
<schedule>30 4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=sunrise]]></url>
<name>nordnUploadSunrise</name>
<description>
This job uploads LORDN Sunrise CSV files for each TLD to MarksDB. It should be
run at most every three hours, or at absolute minimum every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=claims]]></url>
<name>nordnUploadClaims</name>
<description>
This job uploads LORDN Claims CSV files for each TLD to MarksDB. It should be
run at most every three hours, or at absolute minimum every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=sunrise&pullQueue]]></url>
<name>nordnUploadSunrisePullQueue</name>
<description>
This job uploads LORDN Sunrise CSV files for each TLD to MarksDB using
pull queue. It should be run at most every three hours, or at absolute
minimum every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=claims&pullQueue]]></url>
<name>nordnUploadClaimsPullQueue</name>
<description>
This job uploads LORDN Claims CSV files for each TLD to MarksDB using pull
queue. It should be run at most every three hours, or at absolute minimum
every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<name>deleteProberData</name>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>0 14 * * 1</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<name>exportReservedTerms</name>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>30 5 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<name>exportPremiumTerms</name>
<description>
Exports premium price lists to the Google Drive folders for each TLD once per day.
</description>
<schedule>0 5 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<name>readDnsQueue</name>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>*/1 * * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/icannReportingStaging&runInEmpty]]></url>
<name>icannReportingStaging</name>
<description>
Create ICANN activity and transaction reports for last month, storing them in
gs://[PROJECT-ID]-reporting/icann/monthly/yyyy-MM
Upon success, enqueues the icannReportingUpload task to POST these files to ICANN.
</description>
<schedule>0 9 2 * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/icannReportingUpload&runInEmpty]]></url>
<name>icannReportingUpload</name>
<description>
Checks if the monthly ICANN reports have been successfully uploaded. If they have not, attempts to upload them again.
Most of the time, this job should not do anything since the uploads are triggered when the reports are staged.
However, in the event that an upload failed for any reason (e.g. ICANN server is down, IP allow list issues),
this cron job will continue to retry uploads daily until they succeed.
</description>
<schedule>0 15 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/generateInvoices?shouldPublish=true&runInEmpty]]></url>
<name>generateInvoices</name>
<description>
Starts the beam/billing/InvoicingPipeline Dataflow template, which creates the overall invoice and
detail report CSVs for last month, storing them in gs://[PROJECT-ID]-billing/invoices/yyyy-MM.
Upon success, sends an e-mail copy of the invoice to billing personnel, and copies detail
reports to the associated registrars' drive folders.
See GenerateInvoicesAction for more details.
</description>
<!--WARNING: This must occur AFTER expandRecurringBillingEvents and AFTER exportSnapshot, as
it uses Bigquery as the source of truth for billable events. ExportSnapshot usually takes
about 2 hours to complete, so we give 11 hours to be safe. Normally, we give 24+ hours (see
icannReportingStaging), but the invoicing team prefers receiving the e-mail on the first of
each month. -->
<schedule>0 19 1 * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/generateSpec11&runInEmpty]]></url>
<name>generateSpec11</name>
<description>
Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates today's Spec11
report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM/.
This job will only send email notifications on the second of every month.
See GenerateSpec11ReportAction for more details.
</description>
<schedule>0 15 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
<name>wipeOutContactHistoryPii</name>
<description>
This job runs weekly to wipe out PII fields of ContactHistory entities
that have been in the database for a certain period of time.
</description>
<schedule>0 15 * * 1</schedule>
</task>
</taskentries>

312
core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml vendored
View file

@ -1,314 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--TODO: ptkach - Remove this file after cloud scheduler is fully up and running -->
<cronentries>
<!--
/cron/fanout params:
queue=<QUEUE_NAME>
endpoint=<ENDPOINT_NAME> // URL Path of servlet, which may contain placeholders:
runInEmpty // Run once, with no tld parameter
forEachRealTld // Run for tlds with getTldType() == TldType.REAL
forEachTestTld // Run for tlds with getTldType() == TldType.TEST
exclude=TLD1[,TLD2] // exclude something otherwise included
-->
<cron>
<url>/_dr/task/rdeStaging</url>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<!--
This only needs to run once per day, but we launch additional jobs in case the
cursor is lagging behind, so it'll catch up to the current date as quickly as
possible. The only job that'll run under normal circumstances is the one that's
close to midnight, since if the cursor is up-to-date, the task is a no-op.
We want it to be close to midnight because that reduces the chance that the
point-in-time code won't have to go to the extra trouble of fetching old
versions of objects from the database. However, we don't want it to run too
close to midnight, because there's always a chance that a change which was
timestamped before midnight hasn't fully been committed to the database. So
we add a 4+ minute grace period to ensure the transactions cool down, since
our queries are not transactional.
-->
<schedule>every 8 hours from 00:07 to 20:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>every 4 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=rde-report&endpoint=/_dr/task/rdeReport&forEachRealTld]]></url>
<description>
This job is a no-op unless RdeReportCursor falls behind for some reason.
</description>
<schedule>every 4 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>every 12 hours from 00:15 to 12:15</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<!-- TODO(b/249863289): disable until it is safe to run this pipeline
<cron>
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description>
This job resaves all our resources, projected in time to "now".
</description>
<schedule>1st monday of month 09:00</schedule>
<target>backend</target>
</cron>
-->
<cron>
<url><![CDATA[/_dr/task/updateRegistrarRdapBaseUrls]]></url>
<description>
This job reloads all registrar RDAP base URLs from ICANN.
</description>
<schedule>every day 02:34</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>
<description>
This job exports lists of all active domain names to Google Drive and Google Cloud Storage.
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents?advanceCursor]]></url>
<description>
This job runs an action that creates synthetic OneTime billing events from Recurring billing
events. Events are created for all instances of Recurring billing events that should exist
between the RECURRING_BILLING cursor's time and the execution time of the action.
</description>
<schedule>every day 03:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>every day 03:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
<description>
This job runs an action that sends emails to partners if their certificates are expiring soon.
</description>
<schedule>every day 04:30</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=sunrise]]></url>
<description>
This job uploads LORDN Sunrise CSV files for each TLD to MarksDB. It should be
run at most every three hours, or at absolute minimum every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>every 12 hours synchronized</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=claims]]></url>
<description>
This job uploads LORDN Claims CSV files for each TLD to MarksDB. It should be
run at most every three hours, or at absolute minimum every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>every 12 hours synchronized</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=sunrise&pullQueue]]></url>
<description>
This job uploads LORDN Sunrise CSV files for each TLD to MarksDB using
pull queue. It should be run at most every three hours, or at absolute
minimum every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>every 12 hours synchronized</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordnPhase=claims&pullQueue]]></url>
<description>
This job uploads LORDN Claims CSV files for each TLD to MarksDB using pull
queue. It should be run at most every three hours, or at absolute minimum
every 26 hours.
</description>
<!-- This may be set anywhere between "every 3 hours" and "every 25 hours". -->
<schedule>every 12 hours synchronized</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>every monday 14:00</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>every day 05:30</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<description>
Exports premium price lists to the Google Drive folders for each TLD once per day.
</description>
<schedule>every day 05:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>every 1 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/icannReportingStaging&runInEmpty]]></url>
<description>
Create ICANN activity and transaction reports for last month, storing them in
gs://[PROJECT-ID]-reporting/icann/monthly/yyyy-MM
Upon success, enqueues the icannReportingUpload task to POST these files to ICANN.
</description>
<schedule>2 of month 09:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/icannReportingUpload&runInEmpty]]></url>
<description>
Checks if the monthly ICANN reports have been successfully uploaded. If they have not, attempts to upload them again.
Most of the time, this job should not do anything since the uploads are triggered when the reports are staged.
However, in the event that an upload failed for any reason (e.g. ICANN server is down, IP allow list issues),
this cron job will continue to retry uploads daily until they succeed.
</description>
<schedule>every day 15:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/generateInvoices?shouldPublish=true&runInEmpty]]></url>
<description>
Starts the beam/billing/InvoicingPipeline Dataflow template, which creates the overall invoice and
detail report CSVs for last month, storing them in gs://[PROJECT-ID]-billing/invoices/yyyy-MM.
Upon success, sends an e-mail copy of the invoice to billing personnel, and copies detail
reports to the associated registrars' drive folders.
See GenerateInvoicesAction for more details.
</description>
<!--WARNING: This must occur AFTER expandRecurringBillingEvents and AFTER exportSnapshot, as
it uses Bigquery as the source of truth for billable events. ExportSnapshot usually takes
about 2 hours to complete, so we give 11 hours to be safe. Normally, we give 24+ hours (see
icannReportingStaging), but the invoicing team prefers receiving the e-mail on the first of
each month. -->
<schedule>1 of month 19:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/generateSpec11&runInEmpty]]></url>
<description>
Starts the beam/spec11/Spec11Pipeline Dataflow template, which creates today's Spec11
report. This report is stored in gs://[PROJECT-ID]-reporting/icann/spec11/yyyy-MM/.
This job will only send email notifications on the second of every month.
See GenerateSpec11ReportAction for more details.
</description>
<schedule>every day 15:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
<description>
This job runs weekly to wipe out PII fields of ContactHistory entities
that have been in the database for a certain period of time.
</description>
<schedule>every monday 15:00</schedule>
<target>backend</target>
</cron>
</cronentries>

71
core/src/main/java/google/registry/env/qa/default/WEB-INF/cloud-scheduler-tasks.xml vendored Normal file
View file

@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<taskentries>
<task>
<url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<schedule>7 0 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<name>readDnsQueue</name>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>*/1 * * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<name>syncRegistrarsSheet</name>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<name>resaveAllEppResourcesPipeline</name>
<description>
This job resaves all our resources, projected in time to "now".
</description>
<!--Deviation from cron tasks schedule: 1st monday of month 09:00 is replaced
with 1st of the month 09:00 -->
<schedule>0 9 1 * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<name>syncGroupMembers</name>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<name>deleteExpiredDomains</name>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>7 3 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
<name>wipeOutCloudSql</name>
<description>
This job runs an action that deletes all data in Cloud SQL.
</description>
<schedule>7 3 * * 6</schedule>
</task>
</taskentries>

67
core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml vendored
View file

@ -1,70 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--TODO: ptkach - Remove this file after cloud scheduler is fully up and running -->
<cronentries>
<cron>
<url>/_dr/task/rdeStaging</url>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<schedule>every day 00:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>every 1 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description>
This job resaves all our resources, projected in time to "now".
</description>
<schedule>1st monday of month 09:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>every day 03:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
<description>
This job runs an action that deletes all data in Cloud SQL.
</description>
<schedule>every saturday 03:07</schedule>
<target>backend</target>
</cron>
</cronentries>

156
core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml vendored Normal file
View file

@ -0,0 +1,156 @@
<?xml version="1.0" encoding="UTF-8"?>
<taskentries>
<task>
<url>/_dr/task/rdeStaging</url>
<name>rdeStaging</name>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<!--
This only needs to run once per day, but we launch additional jobs in case the
cursor is lagging behind, so it'll catch up to the current date eventually.
See <a href="../../../production/default/WEB-INF/cron.xml">production config</a> for an
explanation of job starting times.
-->
<schedule>7 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<name>rdeUpload</name>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>0 */4 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<name>tmchDnl</name>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<name>tmchSmdrl</name>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>15 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<name>tmchCrl</name>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<name>syncGroupMembers</name>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<name>syncRegistrarsSheet</name>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>0 */1 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>
<name>exportDomainLists</name>
<description>
This job exports lists of all active domain names to Google Drive and Google Cloud Storage.
</description>
<schedule>0 */12 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents?advanceCursor]]></url>
<name>expandRecurringBillingEvents</name>
<description>
This job runs an action that creates synthetic OneTime billing events from Recurring billing
events. Events are created for all instances of Recurring billing events that should exist
between the RECURRING_BILLING cursor's time and the execution time of the action.
</description>
<schedule>0 3 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<name>deleteExpiredDomains</name>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>7 3 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<name>deleteProberData</name>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>0 14 * * 1</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<name>exportReservedTerms</name>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>30 5 * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<name>exportPremiumTerms</name>
<description>
Exports premium price lists to the Google Drive folders for each TLD once per day.
</description>
<schedule>0 5 * * *</schedule>
</task>
<!-- TODO: @ptkach add resaveAllEppResourcesPipelineAction https://cs.opensource.google/nomulus/nomulus/+/master:core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml;l=89 -->
<task>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<name>readDnsQueue</name>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>*/1 * * * *</schedule>
</task>
<task>
<url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
<name>wipeOutContactHistoryPii</name>
<description>
This job runs weekly to wipe out PII fields of ContactHistory entities
that have been in the database for a certain period of time.
</description>
<schedule>0 15 * * 1</schedule>
</task>
</taskentries>

174
core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml vendored
View file

@ -1,177 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--TODO: ptkach - Remove this file after cloud scheduler is fully up and running -->
<cronentries>
<!--
/cron/fanout params:
queue=<QUEUE_NAME>
endpoint=<ENDPOINT_NAME> // URL Path of servlet, which may contain placeholders:
runInEmpty // Run once, with no tld parameter
forEachRealTld // Run for tlds with getTldType() == TldType.REAL
forEachTestTld // Run for tlds with getTldType() == TldType.TEST
exclude=TLD1[,TLD2] // exclude something otherwise included
-->
<cron>
<url>/_dr/task/rdeStaging</url>
<description>
This job generates a full RDE escrow deposit as a single gigantic XML document
and streams it to cloud storage. When this job has finished successfully, it'll
launch a separate task that uploads the deposit file to Iron Mountain via SFTP.
</description>
<!--
This only needs to run once per day, but we launch additional jobs in case the
cursor is lagging behind, so it'll catch up to the current date eventually.
See <a href="../../../production/default/WEB-INF/cron.xml">production config</a> for an
explanation of job starting times.
-->
<schedule>every 12 hours from 00:07 to 12:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
<description>
This job is a no-op unless RdeUploadCursor falls behind for some reason.
</description>
<schedule>every 4 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchDnl&runInEmpty]]></url>
<description>
This job downloads the latest DNL from MarksDB and inserts it into the database.
(See: TmchDnlAction, ClaimsList)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchSmdrl&runInEmpty]]></url>
<description>
This job downloads the latest SMDRL from MarksDB and inserts it into the database.
(See: TmchSmdrlAction, SignedMarkRevocationList)
</description>
<schedule>every 12 hours from 00:15 to 12:15</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=marksdb&endpoint=/_dr/task/tmchCrl&runInEmpty]]></url>
<description>
This job downloads the latest CRL from MarksDB and inserts it into the database.
(See: TmchCrlAction)
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<description>
Syncs RegistrarContact changes in the past hour to Google Groups.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
<description>
Synchronize Registrar entities to Google Spreadsheets.
</description>
<schedule>every 1 hours synchronized</schedule>
<target>backend</target>
</cron>
<!-- TODO(b/249863289): disable until it is safe to run this pipeline
<cron>
<url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description>
This job resaves all our resources, projected in time to "now".
</description>
<schedule>1st monday of month 09:00</schedule>
<target>backend</target>
</cron>
-->
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>
<description>
This job exports lists of all active domain names to Google Drive and Google Cloud Storage.
</description>
<schedule>every 12 hours synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents?advanceCursor]]></url>
<description>
This job runs an action that creates synthetic OneTime billing events from Recurring billing
events. Events are created for all instances of Recurring billing events that should exist
between the RECURRING_BILLING cursor's time and the execution time of the action.
</description>
<schedule>every day 03:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
<description>
This job runs an action that deletes domains that are past their
autorenew end date.
</description>
<schedule>every day 03:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<description>
This job clears out data from probers and runs once a week.
</description>
<schedule>every monday 14:00</schedule>
<timezone>UTC</timezone>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
<description>
Reserved terms export to Google Drive job for creating once-daily exports.
</description>
<schedule>every day 05:30</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportPremiumTerms&forEachRealTld]]></url>
<description>
Exports premium price lists to the Google Drive folders for each TLD once per day.
</description>
<schedule>every day 05:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<description>
Lease all tasks from the dns-pull queue, group by TLD, and invoke PublishDnsUpdates for each
group.
</description>
<schedule>every 1 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
<description>
This job runs weekly to wipe out PII fields of ContactHistory entities
that have been in the database for a certain period of time.
</description>
<schedule>every monday 15:00</schedule>
<target>backend</target>
</cron>
</cronentries>

24
core/src/main/java/google/registry/request/auth/AuthModule.java
View file

@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList;
import dagger.Module;
import dagger.Provides;
import google.registry.config.RegistryConfig.Config;
import javax.inject.Qualifier;
import javax.inject.Singleton;
/**
@ -30,15 +31,26 @@ import javax.inject.Singleton;
public class AuthModule {
private static final String IAP_ISSUER_URL = "https://cloud.google.com/iap";
private static final String SA_ISSUER_URL = "https://accounts.google.com";
/** Provides the custom authentication mechanisms (including OAuth). */
@Provides
ImmutableList<AuthenticationMechanism> provideApiAuthenticationMechanisms(
OAuthAuthenticationMechanism oauthAuthenticationMechanism,
IapHeaderAuthenticationMechanism iapHeaderAuthenticationMechanism) {
return ImmutableList.of(oauthAuthenticationMechanism, iapHeaderAuthenticationMechanism);
IapHeaderAuthenticationMechanism iapHeaderAuthenticationMechanism,
ServiceAccountAuthenticationMechanism serviceAccountAuthenticationMechanism) {
return ImmutableList.of(
oauthAuthenticationMechanism,
iapHeaderAuthenticationMechanism,
serviceAccountAuthenticationMechanism);
}
@Qualifier
@interface IAP {}
@Qualifier
@interface ServiceAccount {}
/** Provides the OAuthService instance. */
@Provides
OAuthService provideOauthService() {
@ -46,10 +58,18 @@ public class AuthModule {
}
@Provides
@IAP
@Singleton
TokenVerifier provideTokenVerifier(
@Config("projectId") String projectId, @Config("projectIdNumber") long projectIdNumber) {
String audience = String.format("/projects/%d/apps/%s", projectIdNumber, projectId);
return TokenVerifier.newBuilder().setAudience(audience).setIssuer(IAP_ISSUER_URL).build();
}
@Provides
@ServiceAccount
@Singleton
TokenVerifier provideServiceAccountTokenVerifier(@Config("projectId") String projectId) {
return TokenVerifier.newBuilder().setAudience(projectId).setIssuer(SA_ISSUER_URL).build();
}
}

47
core/src/main/java/google/registry/request/auth/IapHeaderAuthenticationMechanism.java
View file

@ -14,15 +14,11 @@
package google.registry.request.auth;
import com.google.api.client.json.webtoken.JsonWebSignature;
import com.google.auth.oauth2.TokenVerifier;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryEnvironment;
import google.registry.model.console.User;
import google.registry.model.console.UserDao;
import google.registry.request.auth.AuthModule.IAP;
import java.util.Optional;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
@ -37,41 +33,22 @@ import javax.servlet.http.HttpServletRequest;
* @see <a href="https://cloud.google.com/iap/docs/signed-headers-howto">the documentation on GCP
* IAP's signed headers for more information.</a>
*/
public class IapHeaderAuthenticationMechanism implements AuthenticationMechanism {
public class IapHeaderAuthenticationMechanism extends IdTokenAuthenticationBase {
private static final String ID_TOKEN_HEADER_NAME = "X-Goog-IAP-JWT-Assertion";
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
// A workaround that allows "use" of the IAP-based authenticator when running local testing, i.e.
// the RegistryTestServer
private static Optional<User> userForTesting = Optional.empty();
private final TokenVerifier tokenVerifier;
@Inject
public IapHeaderAuthenticationMechanism(TokenVerifier tokenVerifier) {
this.tokenVerifier = tokenVerifier;
public IapHeaderAuthenticationMechanism(@IAP TokenVerifier tokenVerifier) {
super(tokenVerifier);
}
@Override
public AuthResult authenticate(HttpServletRequest request) {
if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
&& userForTesting.isPresent()) {
return AuthResult.create(AuthLevel.USER, UserAuthInfo.create(userForTesting.get()));
}
String rawIdToken = request.getHeader(ID_TOKEN_HEADER_NAME);
if (rawIdToken == null) {
return AuthResult.NOT_AUTHENTICATED;
}
JsonWebSignature token;
try {
token = tokenVerifier.verify(rawIdToken);
} catch (TokenVerifier.VerificationException e) {
logger.atInfo().withCause(e).log("Error when verifying access token");
return AuthResult.NOT_AUTHENTICATED;
}
String emailAddress = (String) token.getPayload().get("email");
String rawTokenFromRequest(HttpServletRequest request) {
return request.getHeader(ID_TOKEN_HEADER_NAME);
}
@Override
AuthResult authResultFromEmail(String emailAddress) {
Optional<User> maybeUser = UserDao.loadUser(emailAddress);
if (!maybeUser.isPresent()) {
logger.atInfo().log("No user found for email address %s", emailAddress);
@ -80,8 +57,4 @@ public class IapHeaderAuthenticationMechanism implements AuthenticationMechanism
return AuthResult.create(AuthLevel.USER, UserAuthInfo.create(maybeUser.get()));
}
@VisibleForTesting
public static void setUserAuthInfoForTestServer(@Nullable User user) {
userForTesting = Optional.ofNullable(user);
}
}

70
core/src/main/java/google/registry/request/auth/IdTokenAuthenticationBase.java Normal file
View file

@ -0,0 +1,70 @@
// Copyright 2022 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.request.auth;
import com.google.api.client.json.webtoken.JsonWebSignature;
import com.google.auth.oauth2.TokenVerifier;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.FluentLogger;
import google.registry.config.RegistryEnvironment;
import google.registry.model.console.User;
import java.util.Optional;
import javax.annotation.Nullable;
import javax.servlet.http.HttpServletRequest;
public abstract class IdTokenAuthenticationBase implements AuthenticationMechanism {
public static final FluentLogger logger = FluentLogger.forEnclosingClass();
// A workaround that allows "use" of the IAP-based authenticator when running local testing, i.e.
// the RegistryTestServer
private static Optional<User> userForTesting = Optional.empty();
private final TokenVerifier tokenVerifier;
public IdTokenAuthenticationBase(TokenVerifier tokenVerifier) {
this.tokenVerifier = tokenVerifier;
}
abstract String rawTokenFromRequest(HttpServletRequest request);
abstract AuthResult authResultFromEmail(String email);
@Override
public AuthResult authenticate(HttpServletRequest request) {
if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
&& userForTesting.isPresent()) {
return AuthResult.create(AuthLevel.USER, UserAuthInfo.create(userForTesting.get()));
}
String rawIdToken = rawTokenFromRequest(request);
if (rawIdToken == null) {
return AuthResult.NOT_AUTHENTICATED;
}
JsonWebSignature token;
try {
token = tokenVerifier.verify(rawIdToken);
} catch (TokenVerifier.VerificationException e) {
logger.atInfo().withCause(e).log("Error when verifying access token");
return AuthResult.NOT_AUTHENTICATED;
}
String emailAddress = (String) token.getPayload().get("email");
return authResultFromEmail(emailAddress);
}
@VisibleForTesting
public static void setUserAuthInfoForTestServer(@Nullable User user) {
userForTesting = Optional.ofNullable(user);
}
}

62
core/src/main/java/google/registry/request/auth/ServiceAccountAuthenticationMechanism.java Normal file
View file

@ -0,0 +1,62 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.request.auth;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static google.registry.request.auth.AuthLevel.APP;
import com.google.auth.oauth2.TokenVerifier;
import google.registry.config.RegistryConfig.Config;
import google.registry.request.auth.AuthModule.ServiceAccount;
import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
/**
* A way to authenticate HTTP requests signed by Service Account
*
* <p>Currently used by cloud scheduler service account
*/
public class ServiceAccountAuthenticationMechanism extends IdTokenAuthenticationBase {
private final String cloudSchedulerEmailPrefix;
private static final String BEARER_PREFIX = "Bearer ";
@Inject
public ServiceAccountAuthenticationMechanism(
@ServiceAccount TokenVerifier tokenVerifier,
@Config("cloudSchedulerServiceAccountEmail") String cloudSchedulerEmailPrefix) {
super(tokenVerifier);
this.cloudSchedulerEmailPrefix = cloudSchedulerEmailPrefix;
}
@Override
String rawTokenFromRequest(HttpServletRequest request) {
String rawToken = request.getHeader(AUTHORIZATION);
if (rawToken != null && rawToken.startsWith(BEARER_PREFIX)) {
return rawToken.substring(BEARER_PREFIX.length());
}
return null;
}
@Override
AuthResult authResultFromEmail(String emailAddress) {
if (emailAddress.equals(cloudSchedulerEmailPrefix)) {
return AuthResult.create(APP);
} else {
return AuthResult.NOT_AUTHENTICATED;
}
}
}

75
core/src/test/java/google/registry/request/auth/ServiceAccountAuthenticationMechanismTest.java Normal file
View file

@ -0,0 +1,75 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.request.auth;
import static com.google.common.net.HttpHeaders.AUTHORIZATION;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.when;
import com.google.api.client.googleapis.auth.oauth2.GoogleIdToken.Payload;
import com.google.api.client.json.webtoken.JsonWebSignature;
import com.google.api.client.json.webtoken.JsonWebSignature.Header;
import com.google.auth.oauth2.TokenVerifier;
import javax.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
class ServiceAccountAuthenticationMechanismTest {
@Mock private TokenVerifier tokenVerifier;
@Mock private HttpServletRequest request;
private JsonWebSignature token;
private ServiceAccountAuthenticationMechanism serviceAccountAuthenticationMechanism;
@BeforeEach
void beforeEach() throws Exception {
serviceAccountAuthenticationMechanism =
new ServiceAccountAuthenticationMechanism(tokenVerifier, "sa-prefix@email.com");
when(request.getHeader(AUTHORIZATION)).thenReturn("Bearer jwtValue");
Payload payload = new Payload();
payload.setEmail("sa-prefix@email.com");
token = new JsonWebSignature(new Header(), payload, new byte[0], new byte[0]);
when(tokenVerifier.verify("jwtValue")).thenReturn(token);
}
@Test
void testSuccess_authenticates() throws Exception {
AuthResult authResult = serviceAccountAuthenticationMechanism.authenticate(request);
assertThat(authResult.isAuthenticated()).isTrue();
assertThat(authResult.authLevel()).isEqualTo(AuthLevel.APP);
}
@Test
void testFails_authenticateWrongEmail() throws Exception {
token.getPayload().set("email", "not-service-account-email@email.com");
AuthResult authResult = serviceAccountAuthenticationMechanism.authenticate(request);
assertThat(authResult.isAuthenticated()).isFalse();
}
@Test
void testFails_authenticateWrongHeader() throws Exception {
when(request.getHeader(AUTHORIZATION)).thenReturn("BEARER asd");
AuthResult authResult = serviceAccountAuthenticationMechanism.authenticate(request);
assertThat(authResult.isAuthenticated()).isFalse();
}
}

9
release/builder/Dockerfile
View file

@ -19,7 +19,16 @@
# 3. Google Cloud SDK for generating the WARs.
# 4. Git to manipulate the private and the merged repos.
# 5. Docker to build and push images.
# 6. cloudSchedulerDeployer for deploying cloud scheduler tasks based on config
FROM golang:1.19 as cloudSchedulerBuilder
WORKDIR /usr/src/cloudSchedulerDeployer
RUN go mod init cloudSchedulerDeployer
COPY *.go ./
RUN go build -o /cloudSchedulerDeployer
FROM marketplace.gcr.io/google/debian10
ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8
COPY --from=cloudSchedulerBuilder /cloudSchedulerDeployer /usr/local/bin/cloudSchedulerDeployer
ADD ./build.sh .
RUN ["bash", "./build.sh"]

176
release/builder/cloudSchedulerDeployer.go Normal file
View file

@ -0,0 +1,176 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The cloudScheduler tool allows creating, updating and deleting cloud
// scheduler jobs from xml config file
package main
import (
"encoding/json"
"encoding/xml"
"fmt"
"io"
"log"
"os"
"os/exec"
"strings"
)
func main() {
if len(os.Args) < 3 || os.Args[1] == "" || os.Args[2] == "" {
panic("Error - invalid parameters:\nFirst parameter required - config file path;\nSecond parameter required - project name")
}
// Config file path
configFileLocation := os.Args[1]
// Project name where to submit the tasks
projectName := os.Args[2]
log.Default().Println("Filepath " + configFileLocation)
xmlFile, err := os.Open(configFileLocation)
if err != nil {
panic(err)
}
defer xmlFile.Close()
type Task struct {
XMLName xml.Name `xml:"task"`
URL string `xml:"url"`
Description string `xml:"description"`
Schedule string `xml:"schedule"`
Name string `xml:"name"`
}
type Taskentries struct {
XMLName xml.Name `xml:"taskentries"`
Task []Task `xml:"task"`
}
type ServiceAccount struct {
DisplayName string `json:"displayName"`
Email string `json:"email"`
}
type ExistingJob struct {
Name string `json:"name"`
State string `json:"state"`
}
byteValue, _ := io.ReadAll(xmlFile)
var taskEntries Taskentries
if err := xml.Unmarshal(byteValue, &taskEntries); err != nil {
panic("Failed to unmarshal taskentries: " + err.Error())
}
getArgs := func(taskRecord Task, operationType string, serviceAccountEmail string) []string {
// Cloud Schedule doesn't allow description of more than 499 chars and \n
var description string
if len(taskRecord.Description) > 499 {
description = taskRecord.Description[:499]
} else {
description = taskRecord.Description
}
description = strings.ReplaceAll(description, "\n", " ")
return []string{
"--project", projectName,
"scheduler", "jobs", operationType,
"http", taskRecord.Name,
"--location", "us-central1",
"--schedule", taskRecord.Schedule,
"--uri", fmt.Sprintf("https://backend-dot-%s.appspot.com%s", projectName, taskRecord.URL),
"--description", description,
"--http-method", "get",
"--oidc-service-account-email", serviceAccountEmail,
"--oidc-token-audience", projectName,
}
}
// Get existing jobs from Cloud Scheduler
var allExistingJobs []ExistingJob
cmdGetExistingList := exec.Command("gcloud", "scheduler", "jobs", "list", "--project="+projectName, "--location=us-central1", "--format=json")
cmdGetExistingListOutput, cmdGetExistingListError := cmdGetExistingList.CombinedOutput()
if cmdGetExistingListError != nil {
panic("Can't obtain existing cloud scheduler jobs for " + projectName)
}
err = json.Unmarshal(cmdGetExistingListOutput, &allExistingJobs)
if err != nil {
panic("Failed to parse existing jobs from cloud schedule: " + err.Error())
}
// Sync deleted jobs
enabledExistingJobs := map[string]bool{}
for i := 0; i < len(allExistingJobs); i++ {
jobName := strings.Split(allExistingJobs[i].Name, "jobs/")[1]
toBeDeleted := true
if allExistingJobs[i].State == "ENABLED" {
enabledExistingJobs[jobName] = true
}
for i := 0; i < len(taskEntries.Task); i++ {
if taskEntries.Task[i].Name == jobName {
toBeDeleted = false
break
}
}
if toBeDeleted {
cmdDelete := exec.Command("gcloud", "scheduler", "jobs", "delete", jobName, "--project="+projectName, "--quiet")
cmdDeleteOutput, cmdDeleteError := cmdDelete.CombinedOutput()
log.Default().Println("Deleting cloud scheduler job " + jobName)
if cmdDeleteError != nil {
panic(string(cmdDeleteOutput))
}
}
}
// Find service account email
var serviceAccounts []ServiceAccount
var serviceAccountEmail string
cmdGetServiceAccounts := exec.Command("gcloud", "iam", "service-accounts", "list", "--project="+projectName, "--format=json")
cmdGetServiceAccountsOutput, cmdGetServiceAccountsError := cmdGetServiceAccounts.CombinedOutput()
if cmdGetServiceAccountsError != nil {
panic(cmdGetServiceAccountsError)
}
err = json.Unmarshal(cmdGetServiceAccountsOutput, &serviceAccounts)
if err != nil {
panic(err.Error())
}
for i := 0; i < len(serviceAccounts); i++ {
if serviceAccounts[i].DisplayName == "cloud-scheduler" {
serviceAccountEmail = serviceAccounts[i].Email
break
}
}
if serviceAccountEmail == "" {
panic("Service account for cloud scheduler is not created for " + projectName)
}
// Sync created and updated jobs
for i := 0; i < len(taskEntries.Task); i++ {
cmdType := "update"
if enabledExistingJobs[taskEntries.Task[i].Name] != true {
cmdType = "create"
}
syncCommand := exec.Command("gcloud", getArgs(taskEntries.Task[i], cmdType, serviceAccountEmail)...)
syncCommandOutput, syncCommandError := syncCommand.CombinedOutput()
log.Default().Println(cmdType + " cloud scheduler job " + taskEntries.Task[i].Name)
if syncCommandError != nil {
panic(string(syncCommandOutput))
}
}
}

18
release/cloudbuild-deploy.yaml
View file

@ -28,6 +28,22 @@ steps:
set -e
gcloud secrets versions access latest \
--secret nomulus-tool-cloudbuild-credential > tool-credential.json
# Create/Update cloud scheduler jobs based on a cloud-scheduler-tasks.xml
- name: 'gcr.io/$PROJECT_ID/builder:latest'
entrypoint: /bin/bash
args:
- -c
- |
set -e
gcloud auth activate-service-account --key-file=tool-credential.json
if [ ${_ENV} == production ]; then
project_id="domain-registry"
else
project_id="domain-registry-${_ENV}"
fi
gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
tar -xvf ${_ENV}.tar
cloudSchedulerDeployer default/WEB-INF/cloud-scheduler-tasks.xml $project_id
# Deploy the GAE config files.
# First authorize the gcloud tool to use the credential json file, then
# download and unzip the tarball that contains the relevant config files
@ -43,8 +59,6 @@ steps:
else
project_id="domain-registry-${_ENV}"
fi
gsutil cp gs://$PROJECT_ID-deploy/${TAG_NAME}/${_ENV}.tar .
tar -xvf ${_ENV}.tar
for filename in cron dispatch queue; do
gcloud -q --project $project_id app deploy \
default/WEB-INF/appengine-generated/$filename.yaml