From 5d83b4b9f37cc8fea6f70590f300b323b7ad90df Mon Sep 17 00:00:00 2001 From: Sarah Botwinick Date: Fri, 15 Mar 2024 12:35:27 -0400 Subject: [PATCH] Add flyway fix for Concurrent --- core/build.gradle | 2 ++ core/buildscript-gradle.lockfile | 11 ++++++- core/gradle.lockfile | 2 +- .../tools/DumpGoldenSchemaCommand.java | 5 ++++ db/build.gradle | 5 +++- db/gradle.lockfile | 2 +- .../sql/er_diagram/brief_er_diagram.html | 6 ++-- .../sql/er_diagram/full_er_diagram.html | 30 +++++++++++++++++-- db/src/main/resources/sql/flyway.txt | 1 + ...domain_repo_id_indexes_to_more_tables.sql} | 0 .../resources/sql/schema/nomulus.golden.sql | 14 +++++++++ .../registry/sql/flyway/SchemaTest.java | 5 ++++ gradle.properties | 4 +++ jetty/gradle.lockfile | 1 + services/backend/gradle.lockfile | 1 + services/bsa/gradle.lockfile | 1 + services/default/gradle.lockfile | 1 + services/pubapi/gradle.lockfile | 1 + services/tools/gradle.lockfile | 1 + 19 files changed, 83 insertions(+), 10 deletions(-) rename db/src/main/resources/sql/flyway/{V161__add_domain_repo_id_indexes_to_more_tables.sql => V165__add_domain_repo_id_indexes_to_more_tables.sql} (100%) diff --git a/core/build.gradle b/core/build.gradle index ffce12f5d..479395b1e 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -17,6 +17,7 @@ import java.util.Optional plugins { id 'java-library' + id "org.flywaydb.flyway" version "10.8.1" id 'maven-publish' } @@ -317,6 +318,7 @@ dependencies { // Flyway classes needed to generate the golden file. implementation deps['org.flywaydb:flyway-core'] + implementation deps['org.flywaydb:flyway-database-postgresql'] closureCompiler deps['com.google.javascript:closure-compiler'] } diff --git a/core/buildscript-gradle.lockfile b/core/buildscript-gradle.lockfile index 0d156738b..138f1143d 100644 --- a/core/buildscript-gradle.lockfile +++ b/core/buildscript-gradle.lockfile @@ -1,4 +1,13 @@ # This is a Gradle generated file for dependency locking. # Manual edits can break the build and are not advised. # This file is expected to be part of source control. -empty=classpath +com.fasterxml.jackson.core:jackson-annotations:2.15.2=classpath +com.fasterxml.jackson.core:jackson-core:2.15.2=classpath +com.fasterxml.jackson.core:jackson-databind:2.15.2=classpath +com.fasterxml.jackson.dataformat:jackson-dataformat-toml:2.15.2=classpath +com.fasterxml.jackson:jackson-bom:2.15.2=classpath +com.google.code.gson:gson:2.10.1=classpath +gradle.plugin.org.flywaydb:gradle-plugin-publishing:10.8.1=classpath +org.flywaydb.flyway:org.flywaydb.flyway.gradle.plugin:10.8.1=classpath +org.flywaydb:flyway-core:10.8.1=classpath +empty= diff --git a/core/gradle.lockfile b/core/gradle.lockfile index 13062795f..93653ccc5 100644 --- a/core/gradle.lockfile +++ b/core/gradle.lockfile @@ -407,7 +407,7 @@ org.eclipse.jetty:jetty-session:12.0.7=testCompileClasspath,testRuntimeClasspath org.eclipse.jetty:jetty-util:12.0.7=testCompileClasspath,testRuntimeClasspath org.eclipse.jetty:jetty-xml:12.0.7=testCompileClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -org.flywaydb:flyway-database-postgresql:10.10.0=testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:txw2:2.3.1=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.gwtproject:gwt-user:2.10.0=compileClasspath,deploy_jar,nonprodCompileClasspath,nonprodRuntimeClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/core/src/nonprod/java/google/registry/tools/DumpGoldenSchemaCommand.java b/core/src/nonprod/java/google/registry/tools/DumpGoldenSchemaCommand.java index cd7df1c5d..f079d8d1f 100644 --- a/core/src/nonprod/java/google/registry/tools/DumpGoldenSchemaCommand.java +++ b/core/src/nonprod/java/google/registry/tools/DumpGoldenSchemaCommand.java @@ -19,6 +19,7 @@ import com.beust.jcommander.Parameters; import java.io.IOException; import java.nio.file.Path; import org.flywaydb.core.Flyway; +import org.flywaydb.database.postgresql.PostgreSQLConfigurationExtension; import org.testcontainers.containers.Container; /** @@ -50,6 +51,10 @@ public class DumpGoldenSchemaCommand extends PostgresqlCommand { postgresContainer.getUsername(), postgresContainer.getPassword()) .load(); + + PostgreSQLConfigurationExtension configurationExtension = + flyway.getConfigurationExtension(PostgreSQLConfigurationExtension.class); + configurationExtension.setTransactionalLock(false); flyway.migrate(); String userName = postgresContainer.getUsername(); diff --git a/db/build.gradle b/db/build.gradle index 07e2880de..402084063 100644 --- a/db/build.gradle +++ b/db/build.gradle @@ -147,6 +147,9 @@ publishing { flyway { def accessInfo = project.ext.getJdbcAccessInfo() + pluginConfiguration = [ + postgresqlTransactionalLock: 'false' + ] url = accessInfo.url user = accessInfo.user @@ -160,7 +163,7 @@ dependencies { def deps = rootProject.dependencyMap implementation deps['org.flywaydb:flyway-core'] - runtimeOnly deps['org.flywaydb:flyway-database-postgresql'] + implementation deps['org.flywaydb:flyway-database-postgresql'] runtimeOnly deps['com.google.cloud.sql:postgres-socket-factory'] runtimeOnly deps['org.postgresql:postgresql'] diff --git a/db/gradle.lockfile b/db/gradle.lockfile index a61f6c4cc..f4abd648a 100644 --- a/db/gradle.lockfile +++ b/db/gradle.lockfile @@ -91,7 +91,7 @@ org.checkerframework:checker-qual:3.12.0=checkstyle org.checkerframework:checker-qual:3.33.0=annotationProcessor,errorprone,testAnnotationProcessor org.checkerframework:checker-qual:3.42.0=deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=compileClasspath,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -org.flywaydb:flyway-database-postgresql:10.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,deploy_jar,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.hamcrest:hamcrest-core:1.3=testCompileClasspath,testRuntimeClasspath org.jacoco:org.jacoco.agent:0.8.11=jacocoAgent,jacocoAnt org.jacoco:org.jacoco.ant:0.8.11=jacocoAnt diff --git a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html index e6a61256f..667a66a76 100644 --- a/db/src/main/resources/sql/er_diagram/brief_er_diagram.html +++ b/db/src/main/resources/sql/er_diagram/brief_er_diagram.html @@ -261,11 +261,11 @@ td.section { generated on - 2024-03-11 20:42:54.463293864 + 2024-03-14 23:12:37.761866826 last flyway file - V164__rename_console_user_history_time.sql + V165__add_domain_repo_id_indexes_to_more_tables.sql @@ -280,7 +280,7 @@ td.section { generated by SchemaCrawler 16.10.1 generated on - 2024-03-11 20:42:54.463293864 + 2024-03-14 23:12:37.761866826 diff --git a/db/src/main/resources/sql/er_diagram/full_er_diagram.html b/db/src/main/resources/sql/er_diagram/full_er_diagram.html index 700f6ab84..1a4f057fa 100644 --- a/db/src/main/resources/sql/er_diagram/full_er_diagram.html +++ b/db/src/main/resources/sql/er_diagram/full_er_diagram.html @@ -261,11 +261,11 @@ td.section { </tr> <tr> <td class="property_name">generated on</td> - <td class="property_value">2024-03-11 20:42:51.738018408</td> + <td class="property_value">2024-03-14 23:12:35.464952842</td> </tr> <tr> <td class="property_name">last flyway file</td> - <td id="lastFlywayFile" class="property_value">V164__rename_console_user_history_time.sql</td> + <td id="lastFlywayFile" class="property_value">V165__add_domain_repo_id_indexes_to_more_tables.sql</td> </tr> </tbody> </table> @@ -280,7 +280,7 @@ td.section { <text text-anchor="start" x="4443.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated by</text> <text text-anchor="start" x="4526.5" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">SchemaCrawler 16.10.1</text> <text text-anchor="start" x="4442.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">generated on</text> - <text text-anchor="start" x="4526.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-03-11 20:42:51.738018408</text> + <text text-anchor="start" x="4526.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">2024-03-14 23:12:35.464952842</text> <polygon fill="none" stroke="#888888" points="4439,-4 4439,-44 4726,-44 4726,-4 4439,-4" /> <!-- allocationtoken_a08ccbef --> <g id="node1" class="node"> <title> @@ -8671,6 +8671,18 @@ td.section { <tr> <td colspan="3"></td> </tr> + <tr> + <td colspan="2" class="name">idx69qun5kxt3eux5igrxrqcycv0</td> + <td class="description right">[non-unique index]</td> + </tr> + <tr> + <td class="spacer"></td> + <td class="minwidth">domain_history_domain_repo_id</td> + <td class="minwidth">ascending</td> + </tr> + <tr> + <td colspan="3"></td> + </tr> <tr> <td colspan="2" class="name">ukt2e7ae3t8gcsxd13wjx2ka7ij</td> <td class="description right">[unique index]</td> @@ -10253,6 +10265,18 @@ td.section { <tr> <td colspan="3"></td> </tr> + <tr> + <td colspan="2" class="name">idxf2q9dqj899h1q8lah5y719nxd</td> + <td class="description right">[non-unique index]</td> + </tr> + <tr> + <td class="spacer"></td> + <td class="minwidth">domain_repo_id</td> + <td class="minwidth">ascending</td> + </tr> + <tr> + <td colspan="3"></td> + </tr> <tr> <td colspan="2" class="name">idxe7wu46c7wpvfmfnj4565abibp</td> <td class="description right">[non-unique index]</td> diff --git a/db/src/main/resources/sql/flyway.txt b/db/src/main/resources/sql/flyway.txt index 9f71b5755..0006aa626 100644 --- a/db/src/main/resources/sql/flyway.txt +++ b/db/src/main/resources/sql/flyway.txt @@ -162,3 +162,4 @@ V161__rename_console_epp_history_time.sql V162__rename_console_poc_history_time.sql V163__rename_console_registrar_history_time.sql V164__rename_console_user_history_time.sql +V165__add_domain_repo_id_indexes_to_more_tables.sql diff --git a/db/src/main/resources/sql/flyway/V161__add_domain_repo_id_indexes_to_more_tables.sql b/db/src/main/resources/sql/flyway/V165__add_domain_repo_id_indexes_to_more_tables.sql similarity index 100% rename from db/src/main/resources/sql/flyway/V161__add_domain_repo_id_indexes_to_more_tables.sql rename to db/src/main/resources/sql/flyway/V165__add_domain_repo_id_indexes_to_more_tables.sql diff --git a/db/src/main/resources/sql/schema/nomulus.golden.sql b/db/src/main/resources/sql/schema/nomulus.golden.sql index 44974b6e9..cc0362386 100644 --- a/db/src/main/resources/sql/schema/nomulus.golden.sql +++ b/db/src/main/resources/sql/schema/nomulus.golden.sql @@ -1961,6 +1961,13 @@ CREATE INDEX idx5yqacw829y5bm6f7eajsq1cts ON public."UserUpdateHistory" USING bt CREATE INDEX idx67qwkjtlq5q8dv6egtrtnhqi7 ON public."HostHistory" USING btree (history_modification_time); +-- +-- Name: idx69qun5kxt3eux5igrxrqcycv0; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx69qun5kxt3eux5igrxrqcycv0 ON public."DomainHistoryHost" USING btree (domain_history_domain_repo_id); + + -- -- Name: idx6ebt3nwk5ocvnremnhnlkl6ff; Type: INDEX; Schema: public; Owner: - -- @@ -2129,6 +2136,13 @@ CREATE INDEX idxe7wu46c7wpvfmfnj4565abibp ON public."PollMessage" USING btree (r CREATE INDEX idxeokttmxtpq2hohcioe5t2242b ON public."BillingCancellation" USING btree (registrar_id); +-- +-- Name: idxf2q9dqj899h1q8lah5y719nxd; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idxf2q9dqj899h1q8lah5y719nxd ON public."PollMessage" USING btree (domain_repo_id); + + -- -- Name: idxfdk2xpil2x1gh0omt84k2y3o1; Type: INDEX; Schema: public; Owner: - -- diff --git a/db/src/test/java/google/registry/sql/flyway/SchemaTest.java b/db/src/test/java/google/registry/sql/flyway/SchemaTest.java index b4fc834d9..6febe059e 100644 --- a/db/src/test/java/google/registry/sql/flyway/SchemaTest.java +++ b/db/src/test/java/google/registry/sql/flyway/SchemaTest.java @@ -26,6 +26,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import org.flywaydb.core.Flyway; +import org.flywaydb.database.postgresql.PostgreSQLConfigurationExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledIfSystemProperty; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; @@ -92,6 +93,10 @@ class SchemaTest { sqlContainer.getJdbcUrl(), sqlContainer.getUsername(), sqlContainer.getPassword()) .load(); + PostgreSQLConfigurationExtension configurationExtension = + flyway.getConfigurationExtension(PostgreSQLConfigurationExtension.class); + configurationExtension.setTransactionalLock(false); + // flyway.migrate() returns the number of newly pushed scripts. This is a variable // number as our schema evolves. assertThat(flyway.migrate().migrations).isNotEmpty(); diff --git a/gradle.properties b/gradle.properties index 8a8b84822..7d3341d4b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,8 @@ +<<<<<<< HEAD # This file defines properties used by the gradle build. It must be kept in +======= +# This file defines properties used by the gradle build. It must be kept in +>>>>>>> c410b233f (Add flyway fix for Concurrent) # sync with config/nom_build.py. # # To regenerate, run ./nom_build --generate-gradle-properties diff --git a/jetty/gradle.lockfile b/jetty/gradle.lockfile index 318fe72bd..329100757 100644 --- a/jetty/gradle.lockfile +++ b/jetty/gradle.lockfile @@ -324,6 +324,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=deploy_jar,runtimeClasspath,testR org.codehaus.mojo:animal-sniffer-annotations:1.23=deploy_jar,runtimeClasspath,testRuntimeClasspath org.conscrypt:conscrypt-openjdk-uber:2.5.2=deploy_jar,runtimeClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath org.glassfish.jaxb:jaxb-runtime:2.3.1=deploy_jar,runtimeClasspath,testRuntimeClasspath org.glassfish.jaxb:txw2:2.3.1=deploy_jar,runtimeClasspath,testRuntimeClasspath org.gwtproject:gwt-user:2.10.0=deploy_jar,runtimeClasspath,testRuntimeClasspath diff --git a/services/backend/gradle.lockfile b/services/backend/gradle.lockfile index 90a39b3f7..eb3afaf23 100644 --- a/services/backend/gradle.lockfile +++ b/services/backend/gradle.lockfile @@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/services/bsa/gradle.lockfile b/services/bsa/gradle.lockfile index 90a39b3f7..eb3afaf23 100644 --- a/services/bsa/gradle.lockfile +++ b/services/bsa/gradle.lockfile @@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/services/default/gradle.lockfile b/services/default/gradle.lockfile index 90a39b3f7..eb3afaf23 100644 --- a/services/default/gradle.lockfile +++ b/services/default/gradle.lockfile @@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/services/pubapi/gradle.lockfile b/services/pubapi/gradle.lockfile index 90a39b3f7..eb3afaf23 100644 --- a/services/pubapi/gradle.lockfile +++ b/services/pubapi/gradle.lockfile @@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/services/tools/gradle.lockfile b/services/tools/gradle.lockfile index 90a39b3f7..eb3afaf23 100644 --- a/services/tools/gradle.lockfile +++ b/services/tools/gradle.lockfile @@ -304,6 +304,7 @@ org.codehaus.jackson:jackson-mapper-asl:1.9.13=compileClasspath,runtimeClasspath org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath,testRuntimeClasspath org.conscrypt:conscrypt-openjdk-uber:2.5.2=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.flywaydb:flyway-core:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +org.flywaydb:flyway-database-postgresql:10.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:jaxb-runtime:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.glassfish.jaxb:txw2:2.3.1=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath org.gwtproject:gwt-user:2.10.0=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath