Use Flyway to deploy SQL schema to non-prod (#255)

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Corrected the type of ClaimsEntry's revision_id column.
It should be plain int8, not bigserial.

Make GenerateSqlSchemaCommand use a custom dialect that
converts all varchar type to 'text' and timestamp to
'timestamptz'.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.

build.gradle

@@ -82,6 +82,8 @@ apply from: 'dependencies.gradle'
 
 apply from: 'dependency_lic.gradle'
 
+apply from: 'utils.gradle'
+
 // Custom task to run checkLicense in buildSrc, which is not triggered
 // by root project tasks. A shell task is used because buildSrc tasks
 // cannot be referenced in the same way as tasks from a regular included
@@ -236,8 +238,9 @@ subprojects {
     }
   }
 
-  if (['util', 'proxy', 'core', 'prober'].contains(project.name)) return
+  if (['util', 'proxy', 'core', 'prober', 'db'].contains(project.name)) return
 
+  // TODO(weiminyu): investigate if the block below is still needed
   ext.relativePath = "google/registry/${project.name}"
 
   sourceSets.each {
@@ -326,20 +329,6 @@ def createGetBuildSrcDirectDepsTask(outputFileName) {
 }
 
 rootProject.ext {
-
-  // Executes an arbitrary shell command in bash and returns all output
-  // to stdout as a string. This method allows pipes in shell command.
-  execInBash = { shellCommand, bashWorkingDir ->
-    return new ByteArrayOutputStream().withStream { os ->
-      exec {
-        workingDir bashWorkingDir
-        commandLine 'bash', '-c', "${shellCommand}"
-        standardOutput os
-      }
-      return os
-    }.toString().trim()
-  }
-
   invokeJavaDiffFormatScript = { action ->
     def scriptDir = rootDir.path.endsWith('buildSrc')
             ? "${rootDir}/../java-format"

config/dependency-license/allowed_licenses.json

@@ -12,6 +12,9 @@
     {
       "moduleLicense": "Apache-2.0"
     },
+    {
+      "moduleLicense": "Apache License"
+    },
     {
       "moduleLicense": "Apache License 2.0"
     },
@@ -126,9 +129,18 @@
     {
       "moduleLicense": "Eclipse Public License v1.0"
     },
+    {
+      "moduleLicense": "Eclipse Public License - v 2.0"
+    },
+    {
+      "moduleLicense": "https://www.eclipse.org/legal/epl-2.0/, http://www.gnu.org/copyleft/gpl.html, http://www.gnu.org/licenses/lgpl.html"
+    },
     {
       "moduleLicense": "Google App Engine Terms of Service"
     },
+    {
+      "moduleLicense": "GNU General Public License Version 2"
+    },
     {
       "moduleLicense": "GNU General Public License, version 2, with the Classpath Exception"
     },
@@ -144,6 +156,9 @@
     {
       "moduleLicense": "GNU Lesser Public License"
     },
+    {
+      "moduleLicense": "GNU Lesser General Public License Version 2.1"
+    },
     {
       "moduleLicense": "GNU Library General Public License v2.1 or later"
     },

config/presubmits.py

@@ -79,7 +79,8 @@ PRESUBMITS = {
         ("java", "js", "soy", "sql", "py", "sh", "gradle"), {
             ".git", "/build/", "/generated/", "node_modules/",
             "JUnitBackports.java", "registrar_bin.", "registrar_dbg.",
-            "google-java-format-diff.py"
+            "google-java-format-diff.py",
+            "nomulus.golden.sql"
         }, REQUIRED):
         "File did not include the license header.",
 

core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java

@@ -32,11 +32,13 @@ import google.registry.schema.tmch.ClaimsList;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.sql.Types;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
 import org.hibernate.boot.MetadataSources;
 import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
+import org.hibernate.dialect.PostgreSQL95Dialect;
 import org.hibernate.tool.hbm2ddl.SchemaExport;
 import org.hibernate.tool.schema.TargetType;
 import org.joda.time.Period;
@@ -140,7 +142,7 @@ public class GenerateSqlSchemaCommand implements Command {
     try {
       // Configure Hibernate settings.
       Map<String, String> settings = new HashMap<>();
-      settings.put("hibernate.dialect", "org.hibernate.dialect.PostgreSQL9Dialect");
+      settings.put("hibernate.dialect", NomulusPostgreSQLDialect.class.getName());
       settings.put(
           "hibernate.connection.url",
           "jdbc:postgresql://" + databaseHost + ":" + databasePort + "/postgres?useSSL=false");
@@ -190,4 +192,13 @@ public class GenerateSqlSchemaCommand implements Command {
       }
     }
   }
+
+  /** Nomulus mapping rules for column types in Postgresql. */
+  public static class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
+    public NomulusPostgreSQLDialect() {
+      super();
+      registerColumnType(Types.VARCHAR, "text");
+      registerColumnType(Types.TIMESTAMP, "timestamptz");
+    }
+  }
 }

db/README.md

@@ -0,0 +1,47 @@
+## Summary
+
+This project contains Nomulus's Cloud SQL schema and schema deployment utilities.
+
+### Schema Creation DDL
+
+Currently we use Flywaydb for schema deployment. Versioned migration scripts
+are organized in the src/main/resources/sql/flyway folder. Scripts must follow
+the V{id}__{description text}.sql naming pattern (Note the double underscore).
+
+The 'nomulus.golden.sql' file in src/main/resources/sql/schema folder is
+mainly informational. It is generated by Hibernate and should not be
+reformatted. We will use it in validation tests later.
+
+### Non-production Schema Push
+
+To manage schema in a non-production environment, use the 'flywayMigration' task.
+You will need Cloud SDK and login once.
+
+```shell
+# One time login
+gcloud auth login
+
+# Deploy the current schema to alpha
+gradlew :db:flywayMigrate -PdbServer=alpha
+
+# Delete the entire schema in alpha
+gradlew :db:flywayClean -PdbServer=alpha
+```
+
+The flywayMigrate task is idempotent. Repeated runs will not introduce problems.
+
+The Flyway tasks may also be used to deploy to local instances, e.g, your own
+test instance. E.g.,
+
+```shell
+# Deploy to a local instance at standard port as the super user.
+gradlew :db:flywayMigrate -PdbServer=192.168.9.2 -PdbPassword=domain-registry
+
+# Full specification of all parameters
+gradlew :db:flywayMigrate -PdbServer=192.168.9.2:5432 -PdbUser=postgres \
+    -PdbPassword=domain-registry
+```
+
+### Production Schema Deployment
+
+Schema deployment to production and sandbox is under development.

db/build.gradle

@@ -0,0 +1,99 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import com.google.common.collect.ImmutableList
+
+plugins {
+  id "org.flywaydb.flyway" version "6.0.1"
+}
+
+ext {
+  def dbServerProperty = 'dbServer'
+  def dbNameProperty = 'dbName'
+
+  def dbServer = findProperty(dbServerProperty)
+  def dbName = findProperty(dbNameProperty)
+
+  getAccessInfoByHostPort = { hostAndPort ->
+    return [
+        url: "jdbc:postgresql://${hostAndPort}/${dbName}",
+        user: findProperty('dbUser'),
+        password: findProperty('dbPassword')]
+  }
+
+  getSocketFactoryAccessInfo = {
+    def cred = getCloudSqlCredential('alpha', 'superuser').split(' ')
+    def sqlInstance = cred[0]
+    return [
+        url: """\
+                   jdbc:postgresql://google/${dbName}?cloudSqlInstance=
+                   ${sqlInstance}&socketFactory=
+                   com.google.cloud.sql.postgres.SocketFactory"""
+                   .stripIndent()
+                   .replaceAll(System.lineSeparator(), '') ,
+        user: cred[1],
+        password: cred[2]]
+  }
+
+  getJdbcAccessInfo = {
+    switch (dbServer.toString().toLowerCase()) {
+      case 'alpha':
+        return getSocketFactoryAccessInfo()
+      default:
+        return getAccessInfoByHostPort(dbServer)
+    }
+  }
+
+  // Retrieves Cloud SQL credential for a given role. Result is in the form of
+  // 'instancename username password'.
+  //
+  // The env parameter may be one of the following: alpha, crash, sandbox, or
+  // production. The role parameter may be superuser. (More roles will be added
+  // later).
+  getCloudSqlCredential = { env, role ->
+    env = env == 'production' ? '' : "-${env}"
+    def command =
+        """gsutil cp \
+           gs://domain-registry${env}-cloudsql-credentials/${role}.enc - | \
+           gcloud kms decrypt --location global --keyring nomulus \
+           --key sql-credentials-on-gcs-key --plaintext-file=- \
+           --ciphertext-file=- \
+           --project=domain-registry${env}-keys"""
+
+    return execInBash(command, '/tmp')
+  }
+}
+
+flyway {
+  def accessInfo = project.ext.getJdbcAccessInfo()
+
+  url = accessInfo.url
+  user = accessInfo.user
+  password = accessInfo.password
+  schemas = [ 'public' ]
+
+  locations = [ "classpath:sql/flyway" ]
+}
+
+dependencies {
+  runtimeOnly 'org.flywaydb:flyway-core:5.2.4'
+
+  runtimeOnly 'com.google.cloud.sql:postgres-socket-factory:1.0.12'
+  runtimeOnly 'org.postgresql:postgresql:42.2.5'
+}
+
+// Ensure that resources are rebuilt before running Flyway tasks
+tasks
+    .findAll { task -> task.group.equals('Flyway')}
+    .collect { task -> task.dependsOn('buildNeeded') }

db/gradle/dependency-locks/buildscript-classpath.lockfile

@@ -0,0 +1,6 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+gradle.plugin.com.boxfuse.client:gradle-plugin-publishing:6.0.1
+org.flywaydb.flyway:org.flywaydb.flyway.gradle.plugin:6.0.1
+org.flywaydb:flyway-core:6.0.1

db/src/main/resources/sql/flyway/V1__new_claims_list_and_entry.sql

@@ -0,0 +1,31 @@
+-- Copyright 2019 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+    create table "ClaimsEntry" (
+       revision_id int8 not null,
+        claim_key text not null,
+        domain_label text not null,
+        primary key (revision_id, domain_label)
+    );
+
+    create table "ClaimsList" (
+       revision_id  bigserial not null,
+        creation_timestamp timestamptz not null,
+        primary key (revision_id)
+    );
+
+    alter table "ClaimsEntry"
+       add constraint FKlugn0q07ayrtar87dqi3vs3c8 
+       foreign key (revision_id) 
+       references "ClaimsList";

db/src/main/resources/sql/schema/claims_list.sql

@@ -19,7 +19,7 @@ CREATE TABLE `ClaimsList` (
 );
 
 CREATE TABLE `ClaimsEntry` (
-  revision_id BIGSERIAL NOT NULL,
+  revision_id int8 NOT NULL,
   claim_key TEXT NOT NULL,
   domain_label TEXT NOT NULL,
   PRIMARY KEY (revision_id, domain_label),

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -14,14 +14,14 @@
 
     create table ClaimsEntry (
        revision_id int8 not null,
-        claim_key varchar(255) not null,
+        claim_key text not null,
-        domain_label varchar(255) not null,
+        domain_label text not null,
         primary key (revision_id, domain_label)
     );
 
     create table ClaimsList (
        revision_id  bigserial not null,
-        creation_timestamp timestamp not null,
+        creation_timestamp timestamptz not null,
         primary key (revision_id)
     );
 
@@ -40,38 +40,38 @@
     );
 
     create table domain (
-       repoId varchar(255) not null,
+       repoId text not null,
-        creationClientId varchar(255),
+        creationClientId text,
-        currentSponsorClientId varchar(255),
+        currentSponsorClientId text,
         deletionTime bytea,
-        lastEppUpdateClientId varchar(255),
+        lastEppUpdateClientId text,
         lastEppUpdateTime bytea,
         revisions bytea,
-        auth_info_repo_id varchar(255),
+        auth_info_repo_id text,
-        auth_info_value varchar(255),
+        auth_info_value text,
         autorenewBillingEvent bytea,
         autorenewPollMessage bytea,
         deletePollMessage bytea,
-        fullyQualifiedDomainName varchar(255),
+        fullyQualifiedDomainName text,
-        idnTableName varchar(255),
+        idnTableName text,
         lastTransferTime bytea,
         launch_notice_accepted_time bytea,
         launch_notice_expiration_time bytea,
-        launch_notice_tcn_id varchar(255),
+        launch_notice_tcn_id text,
-        launch_notice_validator_id varchar(255),
+        launch_notice_validator_id text,
         registrationExpirationTime bytea,
-        smdId varchar(255),
+        smdId text,
-        tld varchar(255),
+        tld text,
         transfer_data_server_approve_autorenrew_event bytea,
         transfer_data_server_approve_autorenrew_poll_message bytea,
         transfer_data_server_approve_billing_event bytea,
         unit int4,
         value int4,
-        clientTransactionId varchar(255),
+        clientTransactionId text,
-        serverTransactionId varchar(255),
+        serverTransactionId text,
         transfer_data_registration_expiration_time bytea,
-        gainingClientId varchar(255),
+        gainingClientId text,
-        losingClientId varchar(255),
+        losingClientId text,
         pendingTransferExpirationTime bytea,
         transferRequestTime bytea,
         transferStatus int4,
@@ -79,43 +79,43 @@
     );
 
     create table domain_DelegationSignerData (
-       DomainBase_repoId varchar(255) not null,
+       DomainBase_repoId text not null,
         dsData_keyTag int4 not null,
         primary key (DomainBase_repoId, dsData_keyTag)
     );
 
     create table domain_DesignatedContact (
-       DomainBase_repoId varchar(255) not null,
+       DomainBase_repoId text not null,
         allContacts_contact bytea not null,
         primary key (DomainBase_repoId, allContacts_contact)
     );
 
     create table domain_GracePeriod (
-       DomainBase_repoId varchar(255) not null,
+       DomainBase_repoId text not null,
         gracePeriods_id int8 not null,
         primary key (DomainBase_repoId, gracePeriods_id)
     );
 
     create table DomainBase_nsHosts (
-       DomainBase_repoId varchar(255) not null,
+       DomainBase_repoId text not null,
         nsHosts bytea
     );
 
     create table DomainBase_serverApproveEntities (
-       DomainBase_repoId varchar(255) not null,
+       DomainBase_repoId text not null,
         transfer_data_server_approve_entities bytea
     );
 
     create table DomainBase_subordinateHosts (
-       DomainBase_repoId varchar(255) not null,
+       DomainBase_repoId text not null,
-        subordinateHosts varchar(255)
+        subordinateHosts text
     );
 
     create table GracePeriod (
        id  bigserial not null,
         billingEventOneTime bytea,
         billingEventRecurring bytea,
-        clientId varchar(255),
+        clientId text,
         expirationTime bytea,
         type int4,
         primary key (id)
@@ -124,77 +124,77 @@
     create table PremiumEntry (
        revision_id int8 not null,
         price numeric(19, 2) not null,
-        domain_label varchar(255) not null,
+        domain_label text not null,
         primary key (revision_id, domain_label)
     );
 
     create table PremiumList (
        revision_id  bigserial not null,
-        creation_timestamp timestamp not null,
+        creation_timestamp timestamptz not null,
         currency bytea not null,
         primary key (revision_id)
     );
 
-    alter table domain_DelegationSignerData 
+    alter table if exists domain_DelegationSignerData 
        add constraint UK_q2uk7gpqskey3t2w11w2o7x9f unique (dsData_keyTag);
 
-    alter table domain_DesignatedContact 
+    alter table if exists domain_DesignatedContact 
        add constraint UK_fyc0mfvebhatp6sq8dy4jdx4i unique (allContacts_contact);
 
-    alter table domain_GracePeriod 
+    alter table if exists domain_GracePeriod 
        add constraint UK_74osb0s7br4x734ecpdk8caxx unique (gracePeriods_id);
 
-    alter table ClaimsEntry 
+    alter table if exists ClaimsEntry 
        add constraint FKlugn0q07ayrtar87dqi3vs3c8 
        foreign key (revision_id) 
        references ClaimsList;
 
-    alter table domain_DelegationSignerData 
+    alter table if exists domain_DelegationSignerData 
        add constraint FK6p262lfef34yht2ok65rqfoiy 
        foreign key (dsData_keyTag) 
        references DelegationSignerData;
 
-    alter table domain_DelegationSignerData 
+    alter table if exists domain_DelegationSignerData 
        add constraint FK922bmc01akk5mvypcdhtk3qqv 
        foreign key (DomainBase_repoId) 
        references domain;
 
-    alter table domain_DesignatedContact 
+    alter table if exists domain_DesignatedContact 
        add constraint FKdl5kay2hwlalnwcg12cpy12x9 
        foreign key (allContacts_contact) 
        references DesignatedContact;
 
-    alter table domain_DesignatedContact 
+    alter table if exists domain_DesignatedContact 
        add constraint FKb4nx8xr0n24f521y1i1cvr4f2 
        foreign key (DomainBase_repoId) 
        references domain;
 
-    alter table domain_GracePeriod 
+    alter table if exists domain_GracePeriod 
        add constraint FKbw8o0nti4fevxu4xvu8unj726 
        foreign key (gracePeriods_id) 
        references GracePeriod;
 
-    alter table domain_GracePeriod 
+    alter table if exists domain_GracePeriod 
        add constraint FKle4ms7cufyw4vgn5pn01vwwm7 
        foreign key (DomainBase_repoId) 
        references domain;
 
-    alter table DomainBase_nsHosts 
+    alter table if exists DomainBase_nsHosts 
        add constraint FKblxt8vhg3yblt3grqxovoywwm 
        foreign key (DomainBase_repoId) 
        references domain;
 
-    alter table DomainBase_serverApproveEntities 
+    alter table if exists DomainBase_serverApproveEntities 
        add constraint FKbc3iicw0n8s9cj1lca142i7rc 
        foreign key (DomainBase_repoId) 
        references domain;
 
-    alter table DomainBase_subordinateHosts 
+    alter table if exists DomainBase_subordinateHosts 
        add constraint FK3p6gm6lbx46s41hl9wfme77sr 
        foreign key (DomainBase_repoId) 
        references domain;
 
-    alter table PremiumEntry 
+    alter table if exists PremiumEntry 
        add constraint FKqebdja3jkx9c9cnqnrw9g9ocu 
        foreign key (revision_id) 
        references PremiumList;

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -0,0 +1,18 @@
+
+    create table "ClaimsEntry" (
+       revision_id int8 not null,
+        claim_key text not null,
+        domain_label text not null,
+        primary key (revision_id, domain_label)
+    );
+
+    create table "ClaimsList" (
+       revision_id  bigserial not null,
+        creation_timestamp timestamptz not null,
+        primary key (revision_id)
+    );
+
+    alter table "ClaimsEntry"
+       add constraint FKlugn0q07ayrtar87dqi3vs3c8 
+       foreign key (revision_id) 
+       references "ClaimsList";

gradle.properties

@@ -7,3 +7,11 @@ uploaderMultithreadedUpload=
 verboseTestOutput=false
 flowDocsFile=
 enableDependencyLocking=true
+
+# Cloud SQL properties
+
+# A registry environment name (e.g., 'alpha') or a host[:port] string
+dbServer=
+dbName=postgres
+dbUser=
+dbPassword=

utils.gradle

@@ -0,0 +1,33 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+rootProject.ext {
+
+  // Executes an arbitrary shell command in bash and returns all output
+  // to stdout as a string. This method allows pipes in shell command.
+  execInBash = { shellCommand, bashWorkingDir ->
+    return new ByteArrayOutputStream().withStream { os ->
+      exec {
+        workingDir bashWorkingDir
+        commandLine 'bash', '-c', "${shellCommand}"
+        standardOutput os
+      }.assertNormalExitValue()
+      return os
+    }.toString().trim()
+  }
+
+  findOptionalProperty = { propertyName ->
+    return java.util.Optional.ofNullable(project.findProperty(propertyName))
+  }
+}