mirror of
https://github.com/google/nomulus.git
synced 2025-04-30 03:57:51 +02:00
cf94d69a3e
Attempting to run DeleteOldCommitLogs in prod resulted in a lot of DatastoreTimeoutException errors. The assumption is that attempting to load so many CommitLogManifests (over 200 million of them), when each one has a slight possibility of failure, has a very high probability of error. The shard aborts after 20 of these errors, and by eliminating as many loads as possible and retrying the remaining loads inside a transaction we are effectively eliminating any exceptions "leaking" out to the mapreduce framework, which will hopefully keep us bellow 20. At least, that's our best guess currently as to why the mapreduce fails. EppResources are loaded in the map stage to get the revisions, and CommitLogManifests are only loaded in the reduce stage for sanity check so we don't accidentally delete resources we need in prod. Both of these are wrapped in transactNew to make sure they retry individually. The only "load" not done inside a transaction is the EppResourceIndex, but there's no getting around that without rewriting the EppResourceInputs. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=164176764
56 lines
2 KiB
Java
56 lines
2 KiB
Java
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package google.registry.mapreduce.inputs;
|
|
|
|
import com.google.appengine.tools.mapreduce.Input;
|
|
import com.google.appengine.tools.mapreduce.InputReader;
|
|
import com.google.common.base.Optional;
|
|
import com.google.common.collect.ImmutableList;
|
|
import com.googlecode.objectify.Key;
|
|
import google.registry.model.ofy.CommitLogBucket;
|
|
import google.registry.model.ofy.CommitLogManifest;
|
|
import java.util.List;
|
|
import org.joda.time.DateTime;
|
|
|
|
/** Base class for {@link Input} classes that map over {@link CommitLogManifest}. */
|
|
public class CommitLogManifestInput extends Input<Key<CommitLogManifest>> {
|
|
|
|
private static final long serialVersionUID = 2043552272352286428L;
|
|
|
|
/**
|
|
* Cutoff date for result.
|
|
*
|
|
* If present, all resulting CommitLogManifest will be dated prior to this date.
|
|
*/
|
|
private final Optional<DateTime> olderThan;
|
|
|
|
public CommitLogManifestInput(Optional<DateTime> olderThan) {
|
|
this.olderThan = olderThan;
|
|
}
|
|
|
|
@Override
|
|
public List<InputReader<Key<CommitLogManifest>>> createReaders() {
|
|
ImmutableList.Builder<InputReader<Key<CommitLogManifest>>> readers =
|
|
new ImmutableList.Builder<>();
|
|
for (Key<CommitLogBucket> bucketKey : CommitLogBucket.getAllBucketKeys()) {
|
|
readers.add(bucketToReader(bucketKey));
|
|
}
|
|
return readers.build();
|
|
}
|
|
|
|
private InputReader<Key<CommitLogManifest>> bucketToReader(Key<CommitLogBucket> bucketKey) {
|
|
return new CommitLogManifestReader(bucketKey, olderThan);
|
|
}
|
|
}
|