zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-07-20 09:46:03 +02:00
Code Issues Projects Releases Packages Wiki Activity

Remove cron entries for Datastore-specific jobs (#1615)

Browse source 
We'll delete the associated code soon enough too, but it's safer to delete the
cron jobs first and run in that state for a week, so we can still run them
manually if need be.
This commit is contained in:
Ben McIlwain 2022-05-09 17:52:34 -04:00 • committed by GitHub
parent 17d0080742
commit 60b80121d0
6 changed files with 7 additions and 377 deletions
Download patch file Download diff file Expand all files Collapse all files

3
core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java
View file

@ -49,6 +49,7 @@ import google.registry.batch.AsyncTaskMetrics.OperationResult;
import google.registry.dns.DnsQueue; import google.registry.dns.DnsQueue;
import google.registry.mapreduce.MapreduceRunner; import google.registry.mapreduce.MapreduceRunner;
import google.registry.mapreduce.inputs.NullInput; import google.registry.mapreduce.inputs.NullInput;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.domain.DomainBase; import google.registry.model.domain.DomainBase;
import google.registry.model.host.HostResource; import google.registry.model.host.HostResource;
import google.registry.model.server.Lock; import google.registry.model.server.Lock;
@ -79,6 +80,8 @@ import org.joda.time.Duration;
service = Action.Service.BACKEND, service = Action.Service.BACKEND,
path = "/_dr/task/refreshDnsOnHostRename", path = "/_dr/task/refreshDnsOnHostRename",
auth = Auth.AUTH_INTERNAL_OR_ADMIN) auth = Auth.AUTH_INTERNAL_OR_ADMIN)
@DeleteAfterMigration
@Deprecated
public class RefreshDnsOnHostRenameAction implements Runnable { public class RefreshDnsOnHostRenameAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final FluentLogger logger = FluentLogger.forEnclosingClass();

80
core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml vendored
View file

@ -80,7 +80,7 @@
</cron> </cron>
<cron> <cron>
<url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url> <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description> <description>
This job resaves all our resources, projected in time to "now". This job resaves all our resources, projected in time to "now".
It is needed for "deleteOldCommitLogs" to work correctly. It is needed for "deleteOldCommitLogs" to work correctly.
@ -89,48 +89,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/deleteOldCommitLogs]]></url>
<description>
This job deletes unreferenced commit logs from Datastore that are older than thirty days.
Since references are only updated on save, if we want to delete "unneeded" commit logs, we
also need "resaveAllEppResources" to run periodically.
</description>
<schedule>3rd monday of month 09:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
<description>
This job checkpoints the commit log buckets and exports the diff since last checkpoint to GCS.
</description>
<schedule>every 3 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/deleteContactsAndHosts]]></url>
<description>
This job runs a mapreduce that processes batch asynchronous deletions of
contact and host resources by mapping over all EppResources and checking
for any references to the contacts/hosts in pending deletion.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/refreshDnsOnHostRename]]></url>
<description>
This job runs a mapreduce that asynchronously handles DNS refreshes for
host renames by mapping over all domains and creating DNS refresh tasks
for any domains that reference a renamed host.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url> <url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url>
<description> <description>
@ -152,23 +110,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
<description>
This job fires off a Datastore managed-export job that generates snapshot files in GCS.
It also enqueues a new task to wait on the completion of that job and then load the resulting
snapshot into bigquery.
</description>
<!--
Keep google.registry.export.CheckBackupAction.MAXIMUM_BACKUP_RUNNING_TIME less than
this interval. -->
<schedule>every day 06:00</schedule>
<target>backend</target>
</cron>
<!--
Removed for the duration of load testing
TODO(b/71607184): Restore after loadtesting is done
<cron> <cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url> <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<description> <description>
@ -178,7 +119,6 @@
<timezone>UTC</timezone> <timezone>UTC</timezone>
<target>backend</target> <target>backend</target>
</cron> </cron>
-->
<!-- TODO: Add borgmon job to check that these files are created and updated successfully. --> <!-- TODO: Add borgmon job to check that these files are created and updated successfully. -->
<cron> <cron>
@ -199,24 +139,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
<description>
Replays recent commit logs from Datastore to the SQL secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
<description>
Replays recent transactions from SQL to the Datastore secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url> <url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
<description> <description>

78
core/src/main/java/google/registry/env/crash/default/WEB-INF/cron.xml vendored
View file

@ -102,55 +102,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<!-- Disabled for sql-only tests.
<cron>
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
<description>
This job checkpoints the commit log buckets and exports the diff since last checkpoint to GCS.
</description>
<schedule>every 3 minutes synchronized</schedule>
<target>backend</target>
</cron>
-->
<cron>
<url><![CDATA[/_dr/task/deleteContactsAndHosts]]></url>
<description>
This job runs a mapreduce that processes batch asynchronous deletions of
contact and host resources by mapping over all EppResources and checking
for any references to the contacts/hosts in pending deletion.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/refreshDnsOnHostRename]]></url>
<description>
This job runs a mapreduce that asynchronously handles DNS refreshes for
host renames by mapping over all domains and creating DNS refresh tasks
for any domains that reference a renamed host.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<!-- Disabled for sql-only tests
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
<description>
This job fires off a Datastore managed-export job that generates snapshot files in GCS.
It also enqueues a new task to wait on the completion of that job and then load the resulting
snapshot into bigquery.
</description>
<!- -
Keep google.registry.export.CheckBackupAction.MAXIMUM_BACKUP_RUNNING_TIME less than
this interval. - ->
<schedule>every day 06:00</schedule>
<target>backend</target>
</cron>
-->
<cron> <cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url> <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<description> <description>
@ -200,26 +151,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<!-- Disabled for sql-only tests.
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
<description>
Replays recent commit logs from Datastore to the SQL secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
-->
<cron>
<url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
<description>
Replays recent transactions from SQL to the Datastore secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<!-- <!--
The next two wipeout jobs are required when crash has production data. The next two wipeout jobs are required when crash has production data.
--> -->
@ -231,13 +162,4 @@
<schedule>every saturday 03:07</schedule> <schedule>every saturday 03:07</schedule>
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/wipeOutDatastore]]></url>
<description>
This job runs an action that deletes all data in Cloud Datastore.
</description>
<schedule>every saturday 03:07</schedule>
<target>backend</target>
</cron>
</cronentries> </cronentries>

76
core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml vendored
View file

@ -103,7 +103,7 @@
</cron> </cron>
<cron> <cron>
<url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url> <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description> <description>
This job resaves all our resources, projected in time to "now". This job resaves all our resources, projected in time to "now".
It is needed for "deleteOldCommitLogs" to work correctly. It is needed for "deleteOldCommitLogs" to work correctly.
@ -121,26 +121,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/deleteOldCommitLogs]]></url>
<description>
This job deletes unreferenced commit logs from Datastore that are older than thirty days.
Since references are only updated on save, if we want to delete "unneeded" commit logs, we
also need "resaveAllEppResources" to run periodically.
</description>
<schedule>3rd monday of month 09:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
<description>
This job checkpoints the commit log buckets and exports the diff since last checkpoint to GCS.
</description>
<schedule>every 3 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url> <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>
<description> <description>
@ -150,28 +130,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/deleteContactsAndHosts]]></url>
<description>
This job runs a mapreduce that processes batch asynchronous deletions of
contact and host resources by mapping over all EppResources and checking
for any references to the contacts/hosts in pending deletion.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/refreshDnsOnHostRename]]></url>
<description>
This job runs a mapreduce that asynchronously handles DNS refreshes for
host renames by mapping over all domains and creating DNS refresh tasks
for any domains that reference a renamed host.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url> <url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url>
<description> <description>
@ -202,20 +160,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
<description>
This job fires off a Datastore managed-export job that generates snapshot files in GCS.
It also enqueues a new task to wait on the completion of that job and then load the resulting
snapshot into bigquery.
</description>
<!--
Keep google.registry.export.CheckBackupAction.MAXIMUM_BACKUP_RUNNING_TIME less than
this interval. -->
<schedule>every day 06:00</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordn-phase=sunrise]]></url> <url><![CDATA[/_dr/cron/fanout?queue=nordn&endpoint=/_dr/task/nordnUpload&forEachRealTld&lordn-phase=sunrise]]></url>
<description> <description>
@ -340,24 +284,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
<description>
Replays recent commit logs from Datastore to the SQL secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
<description>
Replays recent transactions from SQL to the Datastore secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url> <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
<description> <description>

71
core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml vendored
View file

@ -32,7 +32,7 @@
</cron> </cron>
<cron> <cron>
<url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url> <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description> <description>
This job resaves all our resources, projected in time to "now". This job resaves all our resources, projected in time to "now".
It is needed for "deleteOldCommitLogs" to work correctly. It is needed for "deleteOldCommitLogs" to work correctly.
@ -41,39 +41,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/deleteOldCommitLogs]]></url>
<description>
This job deletes unreferenced commit logs from Datastore that are older than thirty days.
Since references are only updated on save, if we want to delete "unneeded" commit logs, we
also need "resaveAllEppResources" to run periodically.
</description>
<schedule>3rd monday of month 09:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/deleteContactsAndHosts]]></url>
<description>
This job runs a mapreduce that processes batch asynchronous deletions of
contact and host resources by mapping over all EppResources and checking
for any references to the contacts/hosts in pending deletion.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/refreshDnsOnHostRename]]></url>
<description>
This job runs a mapreduce that asynchronously handles DNS refreshes for
host renames by mapping over all domains and creating DNS refresh tasks
for any domains that reference a renamed host.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url> <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/syncGroupMembers&runInEmpty]]></url>
<description> <description>
@ -101,40 +68,4 @@
<schedule>every saturday 03:07</schedule> <schedule>every saturday 03:07</schedule>
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/wipeOutDatastore]]></url>
<description>
This job runs an action that deletes all data in Cloud Datastore.
</description>
<schedule>every saturday 03:07</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
<description>
Replays recent commit logs from Datastore to the SQL secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
<description>
Replays recent transactions from SQL to the Datastore secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
<description>
This job checkpoints the commit log buckets and exports the diff since last checkpoint to GCS.
</description>
<schedule>every 3 minutes synchronized</schedule>
<target>backend</target>
</cron>
</cronentries> </cronentries>

76
core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml vendored
View file

@ -87,7 +87,7 @@
</cron> </cron>
<cron> <cron>
<url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url> <url><![CDATA[/_dr/task/resaveAllEppResourcesPipeline?fast=true]]></url>
<description> <description>
This job resaves all our resources, projected in time to "now". This job resaves all our resources, projected in time to "now".
It is needed for "deleteOldCommitLogs" to work correctly. It is needed for "deleteOldCommitLogs" to work correctly.
@ -96,26 +96,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/deleteOldCommitLogs]]></url>
<description>
This job deletes unreferenced commit logs from Datastore that are older than thirty days.
Since references are only updated on save, if we want to delete "unneeded" commit logs, we
also need "resaveAllEppResources" to run periodically.
</description>
<schedule>3rd monday of month 09:00</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
<description>
This job checkpoints the commit log buckets and exports the diff since last checkpoint to GCS.
</description>
<schedule>every 3 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url> <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportDomainLists&runInEmpty]]></url>
<description> <description>
@ -125,28 +105,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/task/deleteContactsAndHosts]]></url>
<description>
This job runs a mapreduce that processes batch asynchronous deletions of
contact and host resources by mapping over all EppResources and checking
for any references to the contacts/hosts in pending deletion.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/task/refreshDnsOnHostRename]]></url>
<description>
This job runs a mapreduce that asynchronously handles DNS refreshes for
host renames by mapping over all domains and creating DNS refresh tasks
for any domains that reference a renamed host.
</description>
<schedule>every 5 minutes synchronized</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url> <url><![CDATA[/_dr/task/expandRecurringBillingEvents]]></url>
<description> <description>
@ -168,20 +126,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
<description>
This job fires off a Datastore managed-export job that generates snapshot files in GCS.
It also enqueues a new task to wait on the completion of that job and then load the resulting
snapshot into bigquery.
</description>
<!--
Keep google.registry.export.CheckBackupAction.MAXIMUM_BACKUP_RUNNING_TIME less than
this interval. -->
<schedule>every day 06:00</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url> <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
<description> <description>
@ -229,24 +173,6 @@
<target>backend</target> <target>backend</target>
</cron> </cron>
<cron>
<url><![CDATA[/_dr/cron/fanout?queue=replay-commit-logs-to-sql&endpoint=/_dr/task/replayCommitLogsToSql&runInEmpty]]></url>
<description>
Replays recent commit logs from Datastore to the SQL secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron>
<url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
<description>
Replays recent transactions from SQL to the Datastore secondary backend.
</description>
<schedule>every 3 minutes</schedule>
<target>backend</target>
</cron>
<cron> <cron>
<url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url> <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
<description> <description>