mirror of
https://github.com/google/nomulus.git
synced 2025-05-14 16:37:13 +02:00
Migrate to Flogger (green)
This is a 'green' Flogger migration CL. Green CLs are intended to be as safe as possible and should be easy to review and submit. No changes should be necessary to the code itself prior to submission, but small changes to BUILD files may be required. Changes within files are completely independent of each other, so this CL can be safely split up for review using tools such as Rosie. For more information, see [] Base CL: 197826149 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=198560170
This commit is contained in:
jianglai
parent
0d2fb3a8f0
commit
70b13596e4
178 changed files with 984 additions and 988 deletions
|
|
@ -8,7 +8,6 @@ java_library(
|
|||
name = "backup",
|
||||
srcs = glob(["*.java"]),
|
||||
deps = [
|
||||
"//java/com/google/common/logging:formatting_logger",
|
||||
"//java/google/registry/config",
|
||||
"//java/google/registry/cron",
|
||||
"//java/google/registry/mapreduce",
|
||||
|
|
@ -24,6 +23,8 @@ java_library(
|
|||
"@com_google_auto_value",
|
||||
"@com_google_code_findbugs_jsr305",
|
||||
"@com_google_dagger",
|
||||
"@com_google_flogger",
|
||||
"@com_google_flogger_system_backend",
|
||||
"@com_google_guava",
|
||||
"@com_googlecode_json_simple",
|
||||
"@javax_servlet_api",
|
||||
|
|
|
|||
|
|
@ -16,13 +16,12 @@ package google.registry.backup;
|
|||
|
||||
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
|
||||
import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
|
||||
import static com.google.common.logging.FormattingLogger.getLoggerForCallerClass;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.util.DateTimeUtils.isBeforeOrAt;
|
||||
|
||||
import com.google.common.logging.FormattingLogger;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import google.registry.model.ofy.CommitLogCheckpoint;
|
||||
import google.registry.model.ofy.CommitLogCheckpointRoot;
|
||||
import google.registry.request.Action;
|
||||
|
|
@ -50,7 +49,7 @@ import org.joda.time.DateTime;
|
|||
)
|
||||
public final class CommitLogCheckpointAction implements Runnable {
|
||||
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
private static final String QUEUE_NAME = "export-commits";
|
||||
|
||||
|
|
@ -62,13 +61,15 @@ public final class CommitLogCheckpointAction implements Runnable {
|
|||
@Override
|
||||
public void run() {
|
||||
final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
|
||||
logger.infofmt("Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
|
||||
logger.atInfo().log(
|
||||
"Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
|
||||
ofy()
|
||||
.transact(
|
||||
() -> {
|
||||
DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
|
||||
if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
|
||||
logger.infofmt("Newer checkpoint already written at time: %s", lastWrittenTime);
|
||||
logger.atInfo().log(
|
||||
"Newer checkpoint already written at time: %s", lastWrittenTime);
|
||||
return;
|
||||
}
|
||||
ofy()
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ package google.registry.backup;
|
|||
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.logging.FormattingLogger.getLoggerForCallerClass;
|
||||
import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
|
||||
import static google.registry.model.ofy.ObjectifyService.ofy;
|
||||
import static google.registry.util.PipelineUtils.createJobPath;
|
||||
|
|
@ -29,7 +28,7 @@ import com.google.appengine.tools.mapreduce.ReducerInput;
|
|||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMultiset;
|
||||
import com.google.common.logging.FormattingLogger;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.mapreduce.MapreduceRunner;
|
||||
|
|
@ -73,7 +72,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
|||
|
||||
private static final int NUM_MAP_SHARDS = 20;
|
||||
private static final int NUM_REDUCE_SHARDS = 10;
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject MapreduceRunner mrRunner;
|
||||
@Inject Response response;
|
||||
|
|
@ -85,7 +84,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
|||
@Override
|
||||
public void run() {
|
||||
DateTime deletionThreshold = clock.nowUtc().minus(maxAge);
|
||||
logger.infofmt(
|
||||
logger.atInfo().log(
|
||||
"Processing asynchronous deletion of unreferenced CommitLogManifests older than %s",
|
||||
deletionThreshold);
|
||||
|
||||
|
|
@ -182,7 +181,7 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
|||
// First - check if there even are revisions
|
||||
if (eppResource.getRevisions().isEmpty()) {
|
||||
getContext().incrementCounter("EPP resources missing all revisions (SEE LOGS)");
|
||||
logger.severefmt("EPP resource missing all revisions: %s", Key.create(eppResource));
|
||||
logger.atSevere().log("EPP resource missing all revisions: %s", Key.create(eppResource));
|
||||
return;
|
||||
}
|
||||
// Next, check if there's a revision that's older than "CommitLogDatastoreRetention". There
|
||||
|
|
@ -198,9 +197,9 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
|||
}
|
||||
// The oldest revision date is newer than the threshold! This shouldn't happen.
|
||||
getContext().incrementCounter("EPP resources missing pre-threshold revision (SEE LOGS)");
|
||||
logger.severefmt(
|
||||
logger.atSevere().log(
|
||||
"EPP resource missing old enough revision: "
|
||||
+ "%s (created on %s) has %s revisions between %s and %s, while threshold is %s",
|
||||
+ "%s (created on %s) has %d revisions between %s and %s, while threshold is %s",
|
||||
Key.create(eppResource),
|
||||
eppResource.getCreationTime(),
|
||||
eppResource.getRevisions().size(),
|
||||
|
|
@ -320,7 +319,8 @@ public final class DeleteOldCommitLogsAction implements Runnable {
|
|||
getContext().incrementCounter("attempts to delete an already deleted manifest");
|
||||
break;
|
||||
case AFTER_THRESHOLD:
|
||||
logger.severefmt("Won't delete CommitLogManifest %s that is too recent.", manifestKey);
|
||||
logger.atSevere().log(
|
||||
"Won't delete CommitLogManifest %s that is too recent.", manifestKey);
|
||||
getContext().incrementCounter("manifests incorrectly assigned for deletion (SEE LOGS)");
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import static com.google.common.base.Verify.verifyNotNull;
|
|||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static com.google.common.collect.Iterables.concat;
|
||||
import static com.google.common.collect.Lists.partition;
|
||||
import static com.google.common.logging.FormattingLogger.getLoggerForCallerClass;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.LOWER_BOUND_CHECKPOINT;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.NUM_TRANSACTIONS;
|
||||
import static google.registry.backup.BackupUtils.GcsMetadataKeys.UPPER_BOUND_CHECKPOINT;
|
||||
|
|
@ -39,7 +38,7 @@ import com.google.appengine.tools.cloudstorage.GcsService;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.logging.FormattingLogger;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.config.RegistryConfig.Config;
|
||||
import google.registry.model.ImmutableObject;
|
||||
|
|
@ -68,7 +67,7 @@ import org.joda.time.DateTime;
|
|||
)
|
||||
public final class ExportCommitLogDiffAction implements Runnable {
|
||||
|
||||
private static final FormattingLogger logger = getLoggerForCallerClass();
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final String PATH = "/_dr/task/exportCommitLogDiff";
|
||||
static final String UPPER_CHECKPOINT_TIME_PARAM = "upperCheckpointTime";
|
||||
|
|
@ -85,7 +84,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
|||
|
||||
@Override
|
||||
public void run() {
|
||||
logger.infofmt(
|
||||
logger.atInfo().log(
|
||||
"Exporting commit log diffs between %s and %s.", lowerCheckpointTime, upperCheckpointTime);
|
||||
checkArgument(isAtOrAfter(lowerCheckpointTime, START_OF_TIME));
|
||||
checkArgument(lowerCheckpointTime.isBefore(upperCheckpointTime));
|
||||
|
|
@ -99,7 +98,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
|||
|
||||
// Load the keys of all the manifests to include in this diff.
|
||||
List<Key<CommitLogManifest>> sortedKeys = loadAllDiffKeys(lowerCheckpoint, upperCheckpoint);
|
||||
logger.infofmt("Found %d manifests to export", sortedKeys.size());
|
||||
logger.atInfo().log("Found %d manifests to export", sortedKeys.size());
|
||||
// Open an output channel to GCS, wrapped in a stream for convenience.
|
||||
try (OutputStream gcsStream = newOutputStream(gcsService.createOrReplace(
|
||||
new GcsFilename(gcsBucket, DIFF_FILE_PREFIX + upperCheckpointTime),
|
||||
|
|
@ -123,7 +122,7 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
|||
for (int i = 0; i < keyChunks.size(); i++) {
|
||||
// Force the async load to finish.
|
||||
Collection<CommitLogManifest> chunkValues = nextChunkToExport.values();
|
||||
logger.infofmt("Loaded %d manifests", chunkValues.size());
|
||||
logger.atInfo().log("Loaded %d manifests", chunkValues.size());
|
||||
// Since there is no hard bound on how much data this might be, take care not to let the
|
||||
// Objectify session cache fill up and potentially run out of memory. This is the only safe
|
||||
// point to do this since at this point there is no async load in progress.
|
||||
|
|
@ -133,12 +132,12 @@ public final class ExportCommitLogDiffAction implements Runnable {
|
|||
nextChunkToExport = ofy().load().keys(keyChunks.get(i + 1));
|
||||
}
|
||||
exportChunk(gcsStream, chunkValues);
|
||||
logger.infofmt("Exported %d manifests", chunkValues.size());
|
||||
logger.atInfo().log("Exported %d manifests", chunkValues.size());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
logger.infofmt("Exported %d manifests in total", sortedKeys.size());
|
||||
logger.atInfo().log("Exported %d manifests in total", sortedKeys.size());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import com.google.appengine.tools.cloudstorage.GcsService;
|
|||
import com.google.appengine.tools.cloudstorage.ListItem;
|
||||
import com.google.appengine.tools.cloudstorage.ListOptions;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.logging.FormattingLogger;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import com.google.common.util.concurrent.ListeningExecutorService;
|
||||
|
|
@ -44,7 +44,7 @@ import org.joda.time.DateTime;
|
|||
/** Utility class to list commit logs diff files stored on GCS. */
|
||||
class GcsDiffFileLister {
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
@Inject GcsService gcsService;
|
||||
@Inject @Config("commitLogGcsBucket") String gcsBucket;
|
||||
|
|
@ -68,30 +68,29 @@ class GcsDiffFileLister {
|
|||
metadata = Futures.getUnchecked(upperBoundTimesToMetadata.get(checkpointTime));
|
||||
} else {
|
||||
String filename = DIFF_FILE_PREFIX + checkpointTime;
|
||||
logger.infofmt("Patching GCS list; discovered file: %s", filename);
|
||||
logger.atInfo().log("Patching GCS list; discovered file: %s", filename);
|
||||
metadata = getMetadata(filename);
|
||||
|
||||
// If we hit a gap, quit.
|
||||
if (metadata == null) {
|
||||
logger.infofmt(
|
||||
logger.atInfo().log(
|
||||
"Gap discovered in sequence terminating at %s, missing file: %s",
|
||||
sequence.lastKey(),
|
||||
filename);
|
||||
logger.infofmt("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
sequence.lastKey(), filename);
|
||||
logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sequence.put(checkpointTime, metadata);
|
||||
checkpointTime = getLowerBoundTime(metadata);
|
||||
}
|
||||
logger.infofmt("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
logger.atInfo().log("Found sequence from %s to %s", checkpointTime, lastTime);
|
||||
return true;
|
||||
}
|
||||
|
||||
ImmutableList<GcsFileMetadata> listDiffFiles(DateTime fromTime, @Nullable DateTime toTime) {
|
||||
logger.infofmt("Requested restore from time: %s", fromTime);
|
||||
logger.atInfo().log("Requested restore from time: %s", fromTime);
|
||||
if (toTime != null) {
|
||||
logger.infofmt(" Until time: %s", toTime);
|
||||
logger.atInfo().log(" Until time: %s", toTime);
|
||||
}
|
||||
// List all of the diff files on GCS and build a map from each file's upper checkpoint time
|
||||
// (extracted from the filename) to its asynchronously-loaded metadata, keeping only files with
|
||||
|
|
@ -117,7 +116,7 @@ class GcsDiffFileLister {
|
|||
}
|
||||
}
|
||||
if (upperBoundTimesToMetadata.isEmpty()) {
|
||||
logger.info("No files found");
|
||||
logger.atInfo().log("No files found");
|
||||
return ImmutableList.of();
|
||||
}
|
||||
|
||||
|
|
@ -130,7 +129,7 @@ class GcsDiffFileLister {
|
|||
// last file and work backwards we can verify that we have no holes in our chain (although we
|
||||
// may be missing files at the end).
|
||||
TreeMap<DateTime, GcsFileMetadata> sequence = new TreeMap<>();
|
||||
logger.infofmt("Restoring until: %s", lastUpperBoundTime);
|
||||
logger.atInfo().log("Restoring until: %s", lastUpperBoundTime);
|
||||
boolean inconsistentFileSet = !constructDiffSequence(
|
||||
upperBoundTimesToMetadata, fromTime, lastUpperBoundTime, sequence);
|
||||
|
||||
|
|
@ -157,9 +156,9 @@ class GcsDiffFileLister {
|
|||
"Unable to compute commit diff history, there are either gaps or forks in the history "
|
||||
+ "file set. Check log for details.");
|
||||
|
||||
logger.infofmt(
|
||||
logger.atInfo().log(
|
||||
"Actual restore from time: %s", getLowerBoundTime(sequence.firstEntry().getValue()));
|
||||
logger.infofmt("Found %d files to restore", sequence.size());
|
||||
logger.atInfo().log("Found %d files to restore", sequence.size());
|
||||
return ImmutableList.copyOf(sequence.values());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import com.google.appengine.tools.cloudstorage.GcsService;
|
|||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
import com.google.common.collect.Streams;
|
||||
import com.google.common.logging.FormattingLogger;
|
||||
import com.google.common.flogger.FluentLogger;
|
||||
import com.googlecode.objectify.Key;
|
||||
import com.googlecode.objectify.Result;
|
||||
import com.googlecode.objectify.util.ResultNow;
|
||||
|
|
@ -64,7 +64,7 @@ import org.joda.time.DateTime;
|
|||
)
|
||||
public class RestoreCommitLogsAction implements Runnable {
|
||||
|
||||
private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
|
||||
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
|
||||
|
||||
static final int BLOCK_SIZE = 1024 * 1024; // Buffer 1mb at a time, for no particular reason.
|
||||
|
||||
|
|
@ -90,17 +90,17 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
|| RegistryEnvironment.get() == RegistryEnvironment.UNITTEST,
|
||||
"DO NOT RUN ANYWHERE ELSE EXCEPT ALPHA, CRASH OR TESTS.");
|
||||
if (dryRun) {
|
||||
logger.info("Running in dryRun mode");
|
||||
logger.atInfo().log("Running in dryRun mode");
|
||||
}
|
||||
List<GcsFileMetadata> diffFiles = diffLister.listDiffFiles(fromTime, toTime);
|
||||
if (diffFiles.isEmpty()) {
|
||||
logger.info("Nothing to restore");
|
||||
logger.atInfo().log("Nothing to restore");
|
||||
return;
|
||||
}
|
||||
Map<Integer, DateTime> bucketTimestamps = new HashMap<>();
|
||||
CommitLogCheckpoint lastCheckpoint = null;
|
||||
for (GcsFileMetadata metadata : diffFiles) {
|
||||
logger.infofmt("Restoring: %s", metadata.getFilename().getObjectName());
|
||||
logger.atInfo().log("Restoring: %s", metadata.getFilename().getObjectName());
|
||||
try (InputStream input = Channels.newInputStream(
|
||||
gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
|
||||
PeekingIterator<ImmutableObject> commitLogs =
|
||||
|
|
@ -129,7 +129,7 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
.build()),
|
||||
Stream.of(CommitLogCheckpointRoot.create(lastCheckpoint.getCheckpointTime())))
|
||||
.collect(toImmutableList()));
|
||||
logger.info("Restore complete");
|
||||
logger.atInfo().log("Restore complete");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -164,7 +164,7 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
|
||||
private void saveRaw(List<Entity> entitiesToSave) {
|
||||
if (dryRun) {
|
||||
logger.infofmt("Would have saved entities: %s", entitiesToSave);
|
||||
logger.atInfo().log("Would have saved entities: %s", entitiesToSave);
|
||||
return;
|
||||
}
|
||||
retrier.callWithRetry(() -> datastoreService.put(entitiesToSave), RuntimeException.class);
|
||||
|
|
@ -172,7 +172,7 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
|
||||
private void saveOfy(Iterable<? extends ImmutableObject> objectsToSave) {
|
||||
if (dryRun) {
|
||||
logger.infofmt("Would have saved entities: %s", objectsToSave);
|
||||
logger.atInfo().log("Would have saved entities: %s", objectsToSave);
|
||||
return;
|
||||
}
|
||||
retrier.callWithRetry(
|
||||
|
|
@ -181,7 +181,7 @@ public class RestoreCommitLogsAction implements Runnable {
|
|||
|
||||
private Result<?> deleteAsync(Set<Key<?>> keysToDelete) {
|
||||
if (dryRun) {
|
||||
logger.infofmt("Would have deleted entities: %s", keysToDelete);
|
||||
logger.atInfo().log("Would have deleted entities: %s", keysToDelete);
|
||||
}
|
||||
return dryRun || keysToDelete.isEmpty()
|
||||
? new ResultNow<Void>(null)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue