zydronium/google-nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus.git synced 2025-04-29 11:37:51 +02:00
Code Issues Projects Releases Packages Wiki Activity

Use Flyway to deploy SQL schema to non-prod (#255)

Browse source 
* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Corrected the type of ClaimsEntry's revision_id column.
It should be plain int8, not bigserial.

Make GenerateSqlSchemaCommand use a custom dialect that
converts all varchar type to 'text' and timestamp to
'timestamptz'.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.
This commit is contained in:
Weimin Yu 2019-09-06 16:29:49 -04:00 • committed by GitHub
parent c55e3422c2
commit 7f9657469f
13 changed files with 319 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
build.gradle
View file

@ -82,6 +82,8 @@ apply from: 'dependencies.gradle'
apply from: 'dependency_lic.gradle'
apply from: 'utils.gradle'
// Custom task to run checkLicense in buildSrc, which is not triggered
// by root project tasks. A shell task is used because buildSrc tasks
// cannot be referenced in the same way as tasks from a regular included
@ -236,8 +238,9 @@ subprojects {
}
}
if (['util', 'proxy', 'core', 'prober'].contains(project.name)) return
if (['util', 'proxy', 'core', 'prober', 'db'].contains(project.name)) return
// TODO(weiminyu): investigate if the block below is still needed
ext.relativePath = "google/registry/${project.name}"
sourceSets.each {
@ -326,20 +329,6 @@ def createGetBuildSrcDirectDepsTask(outputFileName) {
}
rootProject.ext {
// Executes an arbitrary shell command in bash and returns all output
// to stdout as a string. This method allows pipes in shell command.
execInBash = { shellCommand, bashWorkingDir ->
return new ByteArrayOutputStream().withStream { os ->
exec {
workingDir bashWorkingDir
commandLine 'bash', '-c', "${shellCommand}"
standardOutput os
}
return os
}.toString().trim()
}
invokeJavaDiffFormatScript = { action ->
def scriptDir = rootDir.path.endsWith('buildSrc')
? "${rootDir}/../java-format"

15
config/dependency-license/allowed_licenses.json
View file

@ -12,6 +12,9 @@
{
"moduleLicense": "Apache-2.0"
},
{
"moduleLicense": "Apache License"
},
{
"moduleLicense": "Apache License 2.0"
},
@ -126,9 +129,18 @@
{
"moduleLicense": "Eclipse Public License v1.0"
},
{
"moduleLicense": "Eclipse Public License - v 2.0"
},
{
"moduleLicense": "https://www.eclipse.org/legal/epl-2.0/, http://www.gnu.org/copyleft/gpl.html, http://www.gnu.org/licenses/lgpl.html"
},
{
"moduleLicense": "Google App Engine Terms of Service"
},
{
"moduleLicense": "GNU General Public License Version 2"
},
{
"moduleLicense": "GNU General Public License, version 2, with the Classpath Exception"
},
@ -144,6 +156,9 @@
{
"moduleLicense": "GNU Lesser Public License"
},
{
"moduleLicense": "GNU Lesser General Public License Version 2.1"
},
{
"moduleLicense": "GNU Library General Public License v2.1 or later"
},

3
config/presubmits.py
View file

@ -79,7 +79,8 @@ PRESUBMITS = {
("java", "js", "soy", "sql", "py", "sh", "gradle"), {
".git", "/build/", "/generated/", "node_modules/",
"JUnitBackports.java", "registrar_bin.", "registrar_dbg.",
"google-java-format-diff.py"
"google-java-format-diff.py",
"nomulus.golden.sql"
}, REQUIRED):
"File did not include the license header.",

13
core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java
View file

@ -32,11 +32,13 @@ import google.registry.schema.tmch.ClaimsList;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Types;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import org.hibernate.boot.MetadataSources;
import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
import org.hibernate.dialect.PostgreSQL95Dialect;
import org.hibernate.tool.hbm2ddl.SchemaExport;
import org.hibernate.tool.schema.TargetType;
import org.joda.time.Period;
@ -140,7 +142,7 @@ public class GenerateSqlSchemaCommand implements Command {
try {
// Configure Hibernate settings.
Map<String, String> settings = new HashMap<>();
settings.put("hibernate.dialect", "org.hibernate.dialect.PostgreSQL9Dialect");
settings.put("hibernate.dialect", NomulusPostgreSQLDialect.class.getName());
settings.put(
"hibernate.connection.url",
"jdbc:postgresql://" + databaseHost + ":" + databasePort + "/postgres?useSSL=false");
@ -190,4 +192,13 @@ public class GenerateSqlSchemaCommand implements Command {
}
}
}
/** Nomulus mapping rules for column types in Postgresql. */
public static class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
public NomulusPostgreSQLDialect() {
super();
registerColumnType(Types.VARCHAR, "text");
registerColumnType(Types.TIMESTAMP, "timestamptz");
}
}
}

47
db/README.md Normal file
View file

@ -0,0 +1,47 @@
## Summary
This project contains Nomulus's Cloud SQL schema and schema deployment utilities.
### Schema Creation DDL
Currently we use Flywaydb for schema deployment. Versioned migration scripts
are organized in the src/main/resources/sql/flyway folder. Scripts must follow
the V{id}__{description text}.sql naming pattern (Note the double underscore).
The 'nomulus.golden.sql' file in src/main/resources/sql/schema folder is
mainly informational. It is generated by Hibernate and should not be
reformatted. We will use it in validation tests later.
### Non-production Schema Push
To manage schema in a non-production environment, use the 'flywayMigration' task.
You will need Cloud SDK and login once.
```shell
# One time login
gcloud auth login
# Deploy the current schema to alpha
gradlew :db:flywayMigrate -PdbServer=alpha
# Delete the entire schema in alpha
gradlew :db:flywayClean -PdbServer=alpha
```
The flywayMigrate task is idempotent. Repeated runs will not introduce problems.
The Flyway tasks may also be used to deploy to local instances, e.g, your own
test instance. E.g.,
```shell
# Deploy to a local instance at standard port as the super user.
gradlew :db:flywayMigrate -PdbServer=192.168.9.2 -PdbPassword=domain-registry
# Full specification of all parameters
gradlew :db:flywayMigrate -PdbServer=192.168.9.2:5432 -PdbUser=postgres \
-PdbPassword=domain-registry
```
### Production Schema Deployment
Schema deployment to production and sandbox is under development.

99
db/build.gradle Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import com.google.common.collect.ImmutableList
plugins {
id "org.flywaydb.flyway" version "6.0.1"
}
ext {
def dbServerProperty = 'dbServer'
def dbNameProperty = 'dbName'
def dbServer = findProperty(dbServerProperty)
def dbName = findProperty(dbNameProperty)
getAccessInfoByHostPort = { hostAndPort ->
return [
url: "jdbc:postgresql://${hostAndPort}/${dbName}",
user: findProperty('dbUser'),
password: findProperty('dbPassword')]
}
getSocketFactoryAccessInfo = {
def cred = getCloudSqlCredential('alpha', 'superuser').split(' ')
def sqlInstance = cred[0]
return [
url: """\
jdbc:postgresql://google/${dbName}?cloudSqlInstance=
${sqlInstance}&socketFactory=
com.google.cloud.sql.postgres.SocketFactory"""
.stripIndent()
.replaceAll(System.lineSeparator(), '') ,
user: cred[1],
password: cred[2]]
}
getJdbcAccessInfo = {
switch (dbServer.toString().toLowerCase()) {
case 'alpha':
return getSocketFactoryAccessInfo()
default:
return getAccessInfoByHostPort(dbServer)
}
}
// Retrieves Cloud SQL credential for a given role. Result is in the form of
// 'instancename username password'.
//
// The env parameter may be one of the following: alpha, crash, sandbox, or
// production. The role parameter may be superuser. (More roles will be added
// later).
getCloudSqlCredential = { env, role ->
env = env == 'production' ? '' : "-${env}"
def command =
"""gsutil cp \
gs://domain-registry${env}-cloudsql-credentials/${role}.enc - | \
gcloud kms decrypt --location global --keyring nomulus \
--key sql-credentials-on-gcs-key --plaintext-file=- \
--ciphertext-file=- \
--project=domain-registry${env}-keys"""
return execInBash(command, '/tmp')
}
}
flyway {
def accessInfo = project.ext.getJdbcAccessInfo()
url = accessInfo.url
user = accessInfo.user
password = accessInfo.password
schemas = [ 'public' ]
locations = [ "classpath:sql/flyway" ]
}
dependencies {
runtimeOnly 'org.flywaydb:flyway-core:5.2.4'
runtimeOnly 'com.google.cloud.sql:postgres-socket-factory:1.0.12'
runtimeOnly 'org.postgresql:postgresql:42.2.5'
}
// Ensure that resources are rebuilt before running Flyway tasks
tasks
.findAll { task -> task.group.equals('Flyway')}
.collect { task -> task.dependsOn('buildNeeded') }

6
db/gradle/dependency-locks/buildscript-classpath.lockfile Normal file
View file

@ -0,0 +1,6 @@
# This is a Gradle generated file for dependency locking.
# Manual edits can break the build and are not advised.
# This file is expected to be part of source control.
gradle.plugin.com.boxfuse.client:gradle-plugin-publishing:6.0.1
org.flywaydb.flyway:org.flywaydb.flyway.gradle.plugin:6.0.1
org.flywaydb:flyway-core:6.0.1

31
db/src/main/resources/sql/flyway/V1__new_claims_list_and_entry.sql Normal file
View file

@ -0,0 +1,31 @@
-- Copyright 2019 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
create table "ClaimsEntry" (
revision_id int8 not null,
claim_key text not null,
domain_label text not null,
primary key (revision_id, domain_label)
);
create table "ClaimsList" (
revision_id bigserial not null,
creation_timestamp timestamptz not null,
primary key (revision_id)
);
alter table "ClaimsEntry"
add constraint FKlugn0q07ayrtar87dqi3vs3c8
foreign key (revision_id)
references "ClaimsList";

2
db/src/main/resources/sql/schema/claims_list.sql
View file

@ -19,7 +19,7 @@ CREATE TABLE `ClaimsList` (
);
CREATE TABLE `ClaimsEntry` (
revision_id BIGSERIAL NOT NULL,
revision_id int8 NOT NULL,
claim_key TEXT NOT NULL,
domain_label TEXT NOT NULL,
PRIMARY KEY (revision_id, domain_label),

86
db/src/main/resources/sql/schema/db-schema.sql.generated
View file

@ -14,14 +14,14 @@
create table ClaimsEntry (
revision_id int8 not null,
claim_key varchar(255) not null,
domain_label varchar(255) not null,
claim_key text not null,
domain_label text not null,
primary key (revision_id, domain_label)
);
create table ClaimsList (
revision_id bigserial not null,
creation_timestamp timestamp not null,
creation_timestamp timestamptz not null,
primary key (revision_id)
);
@ -40,38 +40,38 @@
);
create table domain (
repoId varchar(255) not null,
creationClientId varchar(255),
currentSponsorClientId varchar(255),
repoId text not null,
creationClientId text,
currentSponsorClientId text,
deletionTime bytea,
lastEppUpdateClientId varchar(255),
lastEppUpdateClientId text,
lastEppUpdateTime bytea,
revisions bytea,
auth_info_repo_id varchar(255),
auth_info_value varchar(255),
auth_info_repo_id text,
auth_info_value text,
autorenewBillingEvent bytea,
autorenewPollMessage bytea,
deletePollMessage bytea,
fullyQualifiedDomainName varchar(255),
idnTableName varchar(255),
fullyQualifiedDomainName text,
idnTableName text,
lastTransferTime bytea,
launch_notice_accepted_time bytea,
launch_notice_expiration_time bytea,
launch_notice_tcn_id varchar(255),
launch_notice_validator_id varchar(255),
launch_notice_tcn_id text,
launch_notice_validator_id text,
registrationExpirationTime bytea,
smdId varchar(255),
tld varchar(255),
smdId text,
tld text,
transfer_data_server_approve_autorenrew_event bytea,
transfer_data_server_approve_autorenrew_poll_message bytea,
transfer_data_server_approve_billing_event bytea,
unit int4,
value int4,
clientTransactionId varchar(255),
serverTransactionId varchar(255),
clientTransactionId text,
serverTransactionId text,
transfer_data_registration_expiration_time bytea,
gainingClientId varchar(255),
losingClientId varchar(255),
gainingClientId text,
losingClientId text,
pendingTransferExpirationTime bytea,
transferRequestTime bytea,
transferStatus int4,
@ -79,43 +79,43 @@
);
create table domain_DelegationSignerData (
DomainBase_repoId varchar(255) not null,
DomainBase_repoId text not null,
dsData_keyTag int4 not null,
primary key (DomainBase_repoId, dsData_keyTag)
);
create table domain_DesignatedContact (
DomainBase_repoId varchar(255) not null,
DomainBase_repoId text not null,
allContacts_contact bytea not null,
primary key (DomainBase_repoId, allContacts_contact)
);
create table domain_GracePeriod (
DomainBase_repoId varchar(255) not null,
DomainBase_repoId text not null,
gracePeriods_id int8 not null,
primary key (DomainBase_repoId, gracePeriods_id)
);
create table DomainBase_nsHosts (
DomainBase_repoId varchar(255) not null,
DomainBase_repoId text not null,
nsHosts bytea
);
create table DomainBase_serverApproveEntities (
DomainBase_repoId varchar(255) not null,
DomainBase_repoId text not null,
transfer_data_server_approve_entities bytea
);
create table DomainBase_subordinateHosts (
DomainBase_repoId varchar(255) not null,
subordinateHosts varchar(255)
DomainBase_repoId text not null,
subordinateHosts text
);
create table GracePeriod (
id bigserial not null,
billingEventOneTime bytea,
billingEventRecurring bytea,
clientId varchar(255),
clientId text,
expirationTime bytea,
type int4,
primary key (id)
@ -124,77 +124,77 @@
create table PremiumEntry (
revision_id int8 not null,
price numeric(19, 2) not null,
domain_label varchar(255) not null,
domain_label text not null,
primary key (revision_id, domain_label)
);
create table PremiumList (
revision_id bigserial not null,
creation_timestamp timestamp not null,
creation_timestamp timestamptz not null,
currency bytea not null,
primary key (revision_id)
);
alter table domain_DelegationSignerData
alter table if exists domain_DelegationSignerData
add constraint UK_q2uk7gpqskey3t2w11w2o7x9f unique (dsData_keyTag);
alter table domain_DesignatedContact
alter table if exists domain_DesignatedContact
add constraint UK_fyc0mfvebhatp6sq8dy4jdx4i unique (allContacts_contact);
alter table domain_GracePeriod
alter table if exists domain_GracePeriod
add constraint UK_74osb0s7br4x734ecpdk8caxx unique (gracePeriods_id);
alter table ClaimsEntry
alter table if exists ClaimsEntry
add constraint FKlugn0q07ayrtar87dqi3vs3c8
foreign key (revision_id)
references ClaimsList;
alter table domain_DelegationSignerData
alter table if exists domain_DelegationSignerData
add constraint FK6p262lfef34yht2ok65rqfoiy
foreign key (dsData_keyTag)
references DelegationSignerData;
alter table domain_DelegationSignerData
alter table if exists domain_DelegationSignerData
add constraint FK922bmc01akk5mvypcdhtk3qqv
foreign key (DomainBase_repoId)
references domain;
alter table domain_DesignatedContact
alter table if exists domain_DesignatedContact
add constraint FKdl5kay2hwlalnwcg12cpy12x9
foreign key (allContacts_contact)
references DesignatedContact;
alter table domain_DesignatedContact
alter table if exists domain_DesignatedContact
add constraint FKb4nx8xr0n24f521y1i1cvr4f2
foreign key (DomainBase_repoId)
references domain;
alter table domain_GracePeriod
alter table if exists domain_GracePeriod
add constraint FKbw8o0nti4fevxu4xvu8unj726
foreign key (gracePeriods_id)
references GracePeriod;
alter table domain_GracePeriod
alter table if exists domain_GracePeriod
add constraint FKle4ms7cufyw4vgn5pn01vwwm7
foreign key (DomainBase_repoId)
references domain;
alter table DomainBase_nsHosts
alter table if exists DomainBase_nsHosts
add constraint FKblxt8vhg3yblt3grqxovoywwm
foreign key (DomainBase_repoId)
references domain;
alter table DomainBase_serverApproveEntities
alter table if exists DomainBase_serverApproveEntities
add constraint FKbc3iicw0n8s9cj1lca142i7rc
foreign key (DomainBase_repoId)
references domain;
alter table DomainBase_subordinateHosts
alter table if exists DomainBase_subordinateHosts
add constraint FK3p6gm6lbx46s41hl9wfme77sr
foreign key (DomainBase_repoId)
references domain;
alter table PremiumEntry
alter table if exists PremiumEntry
add constraint FKqebdja3jkx9c9cnqnrw9g9ocu
foreign key (revision_id)
references PremiumList;

18
db/src/main/resources/sql/schema/nomulus.golden.sql Normal file
View file

@ -0,0 +1,18 @@
create table "ClaimsEntry" (
revision_id int8 not null,
claim_key text not null,
domain_label text not null,
primary key (revision_id, domain_label)
);
create table "ClaimsList" (
revision_id bigserial not null,
creation_timestamp timestamptz not null,
primary key (revision_id)
);
alter table "ClaimsEntry"
add constraint FKlugn0q07ayrtar87dqi3vs3c8
foreign key (revision_id)
references "ClaimsList";

8
gradle.properties
View file

@ -7,3 +7,11 @@ uploaderMultithreadedUpload=
verboseTestOutput=false
flowDocsFile=
enableDependencyLocking=true
# Cloud SQL properties
# A registry environment name (e.g., 'alpha') or a host[:port] string
dbServer=
dbName=postgres
dbUser=
dbPassword=

33
utils.gradle Normal file
View file

@ -0,0 +1,33 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
rootProject.ext {
// Executes an arbitrary shell command in bash and returns all output
// to stdout as a string. This method allows pipes in shell command.
execInBash = { shellCommand, bashWorkingDir ->
return new ByteArrayOutputStream().withStream { os ->
exec {
workingDir bashWorkingDir
commandLine 'bash', '-c', "${shellCommand}"
standardOutput os
}.assertNormalExitValue()
return os
}.toString().trim()
}
findOptionalProperty = { propertyName ->
return java.util.Optional.ofNullable(project.findProperty(propertyName))
}
}