End-to-end Datastore to SQL pipeline (#707)

* End-to-end Datastore to SQL pipeline

Defined InitSqlPipeline that performs end-to-end migration from
a Datastore backup to a SQL database.

Also fixed/refined multiple tests related to this migration.
This commit is contained in:
Weimin Yu 2020-07-24 09:57:43 -04:00 committed by GitHub
parent 91b7d92cf8
commit 6591e0672a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 1925 additions and 114 deletions

View file

@ -16,6 +16,7 @@ package google.registry.testing.truth;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.truth.Truth.assertAbout;
import static com.google.common.truth.Truth.assertWithMessage;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.github.difflib.DiffUtils;
@ -31,6 +32,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.io.Resources;
import com.google.common.truth.Fact;
import com.google.common.truth.FailureMetadata;
import com.google.common.truth.SimpleSubjectBuilder;
import com.google.common.truth.Subject;
import java.io.IOException;
import java.net.URL;
@ -68,6 +70,15 @@ public class TextDiffSubject extends Subject {
this.actual = ImmutableList.copyOf(actual);
}
protected TextDiffSubject(FailureMetadata metadata, URL actual) {
super(metadata, actual);
try {
this.actual = ImmutableList.copyOf(Resources.asCharSource(actual, UTF_8).readLines());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public TextDiffSubject withDiffFormat(DiffFormat format) {
this.diffFormat = format;
return this;
@ -100,6 +111,11 @@ public class TextDiffSubject extends Subject {
return assertThat(Resources.asCharSource(resourceUrl, UTF_8).readLines());
}
public static SimpleSubjectBuilder<TextDiffSubject, URL> assertWithMessageAboutUrlSource(
String format, Object... params) {
return assertWithMessage(format, params).about(urlFactory());
}
private static final Subject.Factory<TextDiffSubject, ImmutableList<String>>
TEXT_DIFF_SUBJECT_TEXT_FACTORY = TextDiffSubject::new;
@ -107,6 +123,13 @@ public class TextDiffSubject extends Subject {
return TEXT_DIFF_SUBJECT_TEXT_FACTORY;
}
private static final Subject.Factory<TextDiffSubject, URL> TEXT_DIFF_SUBJECT_URL_FACTORY =
TextDiffSubject::new;
public static Subject.Factory<TextDiffSubject, URL> urlFactory() {
return TEXT_DIFF_SUBJECT_URL_FACTORY;
}
static String generateUnifiedDiff(
ImmutableList<String> expectedContent, ImmutableList<String> actualContent) {
Patch<String> diff;