diff --git a/build.gradle b/build.gradle index fe85370be..2fc951aa9 100644 --- a/build.gradle +++ b/build.gradle @@ -82,6 +82,8 @@ apply from: 'dependencies.gradle' apply from: 'dependency_lic.gradle' +apply from: 'utils.gradle' + // Custom task to run checkLicense in buildSrc, which is not triggered // by root project tasks. A shell task is used because buildSrc tasks // cannot be referenced in the same way as tasks from a regular included @@ -236,8 +238,9 @@ subprojects { } } - if (['util', 'proxy', 'core', 'prober'].contains(project.name)) return + if (['util', 'proxy', 'core', 'prober', 'db'].contains(project.name)) return + // TODO(weiminyu): investigate if the block below is still needed ext.relativePath = "google/registry/${project.name}" sourceSets.each { @@ -326,20 +329,6 @@ def createGetBuildSrcDirectDepsTask(outputFileName) { } rootProject.ext { - - // Executes an arbitrary shell command in bash and returns all output - // to stdout as a string. This method allows pipes in shell command. - execInBash = { shellCommand, bashWorkingDir -> - return new ByteArrayOutputStream().withStream { os -> - exec { - workingDir bashWorkingDir - commandLine 'bash', '-c', "${shellCommand}" - standardOutput os - } - return os - }.toString().trim() - } - invokeJavaDiffFormatScript = { action -> def scriptDir = rootDir.path.endsWith('buildSrc') ? "${rootDir}/../java-format" diff --git a/config/dependency-license/allowed_licenses.json b/config/dependency-license/allowed_licenses.json index b88c7d093..9d875d727 100644 --- a/config/dependency-license/allowed_licenses.json +++ b/config/dependency-license/allowed_licenses.json @@ -12,6 +12,9 @@ { "moduleLicense": "Apache-2.0" }, + { + "moduleLicense": "Apache License" + }, { "moduleLicense": "Apache License 2.0" }, @@ -126,9 +129,18 @@ { "moduleLicense": "Eclipse Public License v1.0" }, + { + "moduleLicense": "Eclipse Public License - v 2.0" + }, + { + "moduleLicense": "https://www.eclipse.org/legal/epl-2.0/, http://www.gnu.org/copyleft/gpl.html, http://www.gnu.org/licenses/lgpl.html" + }, { "moduleLicense": "Google App Engine Terms of Service" }, + { + "moduleLicense": "GNU General Public License Version 2" + }, { "moduleLicense": "GNU General Public License, version 2, with the Classpath Exception" }, @@ -144,6 +156,9 @@ { "moduleLicense": "GNU Lesser Public License" }, + { + "moduleLicense": "GNU Lesser General Public License Version 2.1" + }, { "moduleLicense": "GNU Library General Public License v2.1 or later" }, diff --git a/config/presubmits.py b/config/presubmits.py index e30f0dcaf..6f77d0172 100644 --- a/config/presubmits.py +++ b/config/presubmits.py @@ -79,7 +79,8 @@ PRESUBMITS = { ("java", "js", "soy", "sql", "py", "sh", "gradle"), { ".git", "/build/", "/generated/", "node_modules/", "JUnitBackports.java", "registrar_bin.", "registrar_dbg.", - "google-java-format-diff.py" + "google-java-format-diff.py", + "nomulus.golden.sql" }, REQUIRED): "File did not include the license header.", diff --git a/core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java b/core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java index 8541ccb7a..2aa6bfdaf 100644 --- a/core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java +++ b/core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java @@ -32,11 +32,13 @@ import google.registry.schema.tmch.ClaimsList; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; +import java.sql.Types; import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import org.hibernate.boot.MetadataSources; import org.hibernate.boot.registry.StandardServiceRegistryBuilder; +import org.hibernate.dialect.PostgreSQL95Dialect; import org.hibernate.tool.hbm2ddl.SchemaExport; import org.hibernate.tool.schema.TargetType; import org.joda.time.Period; @@ -140,7 +142,7 @@ public class GenerateSqlSchemaCommand implements Command { try { // Configure Hibernate settings. Map settings = new HashMap<>(); - settings.put("hibernate.dialect", "org.hibernate.dialect.PostgreSQL9Dialect"); + settings.put("hibernate.dialect", NomulusPostgreSQLDialect.class.getName()); settings.put( "hibernate.connection.url", "jdbc:postgresql://" + databaseHost + ":" + databasePort + "/postgres?useSSL=false"); @@ -190,4 +192,13 @@ public class GenerateSqlSchemaCommand implements Command { } } } + + /** Nomulus mapping rules for column types in Postgresql. */ + public static class NomulusPostgreSQLDialect extends PostgreSQL95Dialect { + public NomulusPostgreSQLDialect() { + super(); + registerColumnType(Types.VARCHAR, "text"); + registerColumnType(Types.TIMESTAMP, "timestamptz"); + } + } } diff --git a/db/README.md b/db/README.md new file mode 100644 index 000000000..add47e793 --- /dev/null +++ b/db/README.md @@ -0,0 +1,47 @@ +## Summary + +This project contains Nomulus's Cloud SQL schema and schema deployment utilities. + +### Schema Creation DDL + +Currently we use Flywaydb for schema deployment. Versioned migration scripts +are organized in the src/main/resources/sql/flyway folder. Scripts must follow +the V{id}__{description text}.sql naming pattern (Note the double underscore). + +The 'nomulus.golden.sql' file in src/main/resources/sql/schema folder is +mainly informational. It is generated by Hibernate and should not be +reformatted. We will use it in validation tests later. + +### Non-production Schema Push + +To manage schema in a non-production environment, use the 'flywayMigration' task. +You will need Cloud SDK and login once. + +```shell +# One time login +gcloud auth login + +# Deploy the current schema to alpha +gradlew :db:flywayMigrate -PdbServer=alpha + +# Delete the entire schema in alpha +gradlew :db:flywayClean -PdbServer=alpha +``` + +The flywayMigrate task is idempotent. Repeated runs will not introduce problems. + +The Flyway tasks may also be used to deploy to local instances, e.g, your own +test instance. E.g., + +```shell +# Deploy to a local instance at standard port as the super user. +gradlew :db:flywayMigrate -PdbServer=192.168.9.2 -PdbPassword=domain-registry + +# Full specification of all parameters +gradlew :db:flywayMigrate -PdbServer=192.168.9.2:5432 -PdbUser=postgres \ + -PdbPassword=domain-registry +``` + +### Production Schema Deployment + +Schema deployment to production and sandbox is under development. \ No newline at end of file diff --git a/db/build.gradle b/db/build.gradle new file mode 100644 index 000000000..c7e9f9ea4 --- /dev/null +++ b/db/build.gradle @@ -0,0 +1,99 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import com.google.common.collect.ImmutableList + +plugins { + id "org.flywaydb.flyway" version "6.0.1" +} + +ext { + def dbServerProperty = 'dbServer' + def dbNameProperty = 'dbName' + + def dbServer = findProperty(dbServerProperty) + def dbName = findProperty(dbNameProperty) + + getAccessInfoByHostPort = { hostAndPort -> + return [ + url: "jdbc:postgresql://${hostAndPort}/${dbName}", + user: findProperty('dbUser'), + password: findProperty('dbPassword')] + } + + getSocketFactoryAccessInfo = { + def cred = getCloudSqlCredential('alpha', 'superuser').split(' ') + def sqlInstance = cred[0] + return [ + url: """\ + jdbc:postgresql://google/${dbName}?cloudSqlInstance= + ${sqlInstance}&socketFactory= + com.google.cloud.sql.postgres.SocketFactory""" + .stripIndent() + .replaceAll(System.lineSeparator(), '') , + user: cred[1], + password: cred[2]] + } + + getJdbcAccessInfo = { + switch (dbServer.toString().toLowerCase()) { + case 'alpha': + return getSocketFactoryAccessInfo() + default: + return getAccessInfoByHostPort(dbServer) + } + } + + // Retrieves Cloud SQL credential for a given role. Result is in the form of + // 'instancename username password'. + // + // The env parameter may be one of the following: alpha, crash, sandbox, or + // production. The role parameter may be superuser. (More roles will be added + // later). + getCloudSqlCredential = { env, role -> + env = env == 'production' ? '' : "-${env}" + def command = + """gsutil cp \ + gs://domain-registry${env}-cloudsql-credentials/${role}.enc - | \ + gcloud kms decrypt --location global --keyring nomulus \ + --key sql-credentials-on-gcs-key --plaintext-file=- \ + --ciphertext-file=- \ + --project=domain-registry${env}-keys""" + + return execInBash(command, '/tmp') + } +} + +flyway { + def accessInfo = project.ext.getJdbcAccessInfo() + + url = accessInfo.url + user = accessInfo.user + password = accessInfo.password + schemas = [ 'public' ] + + locations = [ "classpath:sql/flyway" ] +} + +dependencies { + runtimeOnly 'org.flywaydb:flyway-core:5.2.4' + + runtimeOnly 'com.google.cloud.sql:postgres-socket-factory:1.0.12' + runtimeOnly 'org.postgresql:postgresql:42.2.5' +} + +// Ensure that resources are rebuilt before running Flyway tasks +tasks + .findAll { task -> task.group.equals('Flyway')} + .collect { task -> task.dependsOn('buildNeeded') } diff --git a/db/gradle/dependency-locks/buildscript-classpath.lockfile b/db/gradle/dependency-locks/buildscript-classpath.lockfile new file mode 100644 index 000000000..12a0e7366 --- /dev/null +++ b/db/gradle/dependency-locks/buildscript-classpath.lockfile @@ -0,0 +1,6 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +gradle.plugin.com.boxfuse.client:gradle-plugin-publishing:6.0.1 +org.flywaydb.flyway:org.flywaydb.flyway.gradle.plugin:6.0.1 +org.flywaydb:flyway-core:6.0.1 diff --git a/db/src/main/resources/sql/flyway/V1__new_claims_list_and_entry.sql b/db/src/main/resources/sql/flyway/V1__new_claims_list_and_entry.sql new file mode 100644 index 000000000..87c69723f --- /dev/null +++ b/db/src/main/resources/sql/flyway/V1__new_claims_list_and_entry.sql @@ -0,0 +1,31 @@ +-- Copyright 2019 The Nomulus Authors. All Rights Reserved. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + + create table "ClaimsEntry" ( + revision_id int8 not null, + claim_key text not null, + domain_label text not null, + primary key (revision_id, domain_label) + ); + + create table "ClaimsList" ( + revision_id bigserial not null, + creation_timestamp timestamptz not null, + primary key (revision_id) + ); + + alter table "ClaimsEntry" + add constraint FKlugn0q07ayrtar87dqi3vs3c8 + foreign key (revision_id) + references "ClaimsList"; diff --git a/db/src/main/resources/sql/schema/claims_list.sql b/db/src/main/resources/sql/schema/claims_list.sql index c73ccadb2..85e3504ee 100644 --- a/db/src/main/resources/sql/schema/claims_list.sql +++ b/db/src/main/resources/sql/schema/claims_list.sql @@ -19,7 +19,7 @@ CREATE TABLE `ClaimsList` ( ); CREATE TABLE `ClaimsEntry` ( - revision_id BIGSERIAL NOT NULL, + revision_id int8 NOT NULL, claim_key TEXT NOT NULL, domain_label TEXT NOT NULL, PRIMARY KEY (revision_id, domain_label), diff --git a/db/src/main/resources/sql/schema/db-schema.sql.generated b/db/src/main/resources/sql/schema/db-schema.sql.generated index d59d2d22a..8a564d216 100644 --- a/db/src/main/resources/sql/schema/db-schema.sql.generated +++ b/db/src/main/resources/sql/schema/db-schema.sql.generated @@ -14,14 +14,14 @@ create table ClaimsEntry ( revision_id int8 not null, - claim_key varchar(255) not null, - domain_label varchar(255) not null, + claim_key text not null, + domain_label text not null, primary key (revision_id, domain_label) ); create table ClaimsList ( revision_id bigserial not null, - creation_timestamp timestamp not null, + creation_timestamp timestamptz not null, primary key (revision_id) ); @@ -40,38 +40,38 @@ ); create table domain ( - repoId varchar(255) not null, - creationClientId varchar(255), - currentSponsorClientId varchar(255), + repoId text not null, + creationClientId text, + currentSponsorClientId text, deletionTime bytea, - lastEppUpdateClientId varchar(255), + lastEppUpdateClientId text, lastEppUpdateTime bytea, revisions bytea, - auth_info_repo_id varchar(255), - auth_info_value varchar(255), + auth_info_repo_id text, + auth_info_value text, autorenewBillingEvent bytea, autorenewPollMessage bytea, deletePollMessage bytea, - fullyQualifiedDomainName varchar(255), - idnTableName varchar(255), + fullyQualifiedDomainName text, + idnTableName text, lastTransferTime bytea, launch_notice_accepted_time bytea, launch_notice_expiration_time bytea, - launch_notice_tcn_id varchar(255), - launch_notice_validator_id varchar(255), + launch_notice_tcn_id text, + launch_notice_validator_id text, registrationExpirationTime bytea, - smdId varchar(255), - tld varchar(255), + smdId text, + tld text, transfer_data_server_approve_autorenrew_event bytea, transfer_data_server_approve_autorenrew_poll_message bytea, transfer_data_server_approve_billing_event bytea, unit int4, value int4, - clientTransactionId varchar(255), - serverTransactionId varchar(255), + clientTransactionId text, + serverTransactionId text, transfer_data_registration_expiration_time bytea, - gainingClientId varchar(255), - losingClientId varchar(255), + gainingClientId text, + losingClientId text, pendingTransferExpirationTime bytea, transferRequestTime bytea, transferStatus int4, @@ -79,43 +79,43 @@ ); create table domain_DelegationSignerData ( - DomainBase_repoId varchar(255) not null, + DomainBase_repoId text not null, dsData_keyTag int4 not null, primary key (DomainBase_repoId, dsData_keyTag) ); create table domain_DesignatedContact ( - DomainBase_repoId varchar(255) not null, + DomainBase_repoId text not null, allContacts_contact bytea not null, primary key (DomainBase_repoId, allContacts_contact) ); create table domain_GracePeriod ( - DomainBase_repoId varchar(255) not null, + DomainBase_repoId text not null, gracePeriods_id int8 not null, primary key (DomainBase_repoId, gracePeriods_id) ); create table DomainBase_nsHosts ( - DomainBase_repoId varchar(255) not null, + DomainBase_repoId text not null, nsHosts bytea ); create table DomainBase_serverApproveEntities ( - DomainBase_repoId varchar(255) not null, + DomainBase_repoId text not null, transfer_data_server_approve_entities bytea ); create table DomainBase_subordinateHosts ( - DomainBase_repoId varchar(255) not null, - subordinateHosts varchar(255) + DomainBase_repoId text not null, + subordinateHosts text ); create table GracePeriod ( id bigserial not null, billingEventOneTime bytea, billingEventRecurring bytea, - clientId varchar(255), + clientId text, expirationTime bytea, type int4, primary key (id) @@ -124,77 +124,77 @@ create table PremiumEntry ( revision_id int8 not null, price numeric(19, 2) not null, - domain_label varchar(255) not null, + domain_label text not null, primary key (revision_id, domain_label) ); create table PremiumList ( revision_id bigserial not null, - creation_timestamp timestamp not null, + creation_timestamp timestamptz not null, currency bytea not null, primary key (revision_id) ); - alter table domain_DelegationSignerData + alter table if exists domain_DelegationSignerData add constraint UK_q2uk7gpqskey3t2w11w2o7x9f unique (dsData_keyTag); - alter table domain_DesignatedContact + alter table if exists domain_DesignatedContact add constraint UK_fyc0mfvebhatp6sq8dy4jdx4i unique (allContacts_contact); - alter table domain_GracePeriod + alter table if exists domain_GracePeriod add constraint UK_74osb0s7br4x734ecpdk8caxx unique (gracePeriods_id); - alter table ClaimsEntry + alter table if exists ClaimsEntry add constraint FKlugn0q07ayrtar87dqi3vs3c8 foreign key (revision_id) references ClaimsList; - alter table domain_DelegationSignerData + alter table if exists domain_DelegationSignerData add constraint FK6p262lfef34yht2ok65rqfoiy foreign key (dsData_keyTag) references DelegationSignerData; - alter table domain_DelegationSignerData + alter table if exists domain_DelegationSignerData add constraint FK922bmc01akk5mvypcdhtk3qqv foreign key (DomainBase_repoId) references domain; - alter table domain_DesignatedContact + alter table if exists domain_DesignatedContact add constraint FKdl5kay2hwlalnwcg12cpy12x9 foreign key (allContacts_contact) references DesignatedContact; - alter table domain_DesignatedContact + alter table if exists domain_DesignatedContact add constraint FKb4nx8xr0n24f521y1i1cvr4f2 foreign key (DomainBase_repoId) references domain; - alter table domain_GracePeriod + alter table if exists domain_GracePeriod add constraint FKbw8o0nti4fevxu4xvu8unj726 foreign key (gracePeriods_id) references GracePeriod; - alter table domain_GracePeriod + alter table if exists domain_GracePeriod add constraint FKle4ms7cufyw4vgn5pn01vwwm7 foreign key (DomainBase_repoId) references domain; - alter table DomainBase_nsHosts + alter table if exists DomainBase_nsHosts add constraint FKblxt8vhg3yblt3grqxovoywwm foreign key (DomainBase_repoId) references domain; - alter table DomainBase_serverApproveEntities + alter table if exists DomainBase_serverApproveEntities add constraint FKbc3iicw0n8s9cj1lca142i7rc foreign key (DomainBase_repoId) references domain; - alter table DomainBase_subordinateHosts + alter table if exists DomainBase_subordinateHosts add constraint FK3p6gm6lbx46s41hl9wfme77sr foreign key (DomainBase_repoId) references domain; - alter table PremiumEntry + alter table if exists PremiumEntry add constraint FKqebdja3jkx9c9cnqnrw9g9ocu foreign key (revision_id) references PremiumList; diff --git a/db/src/main/resources/sql/schema/nomulus.golden.sql b/db/src/main/resources/sql/schema/nomulus.golden.sql new file mode 100644 index 000000000..5ec31dd06 --- /dev/null +++ b/db/src/main/resources/sql/schema/nomulus.golden.sql @@ -0,0 +1,18 @@ + + create table "ClaimsEntry" ( + revision_id int8 not null, + claim_key text not null, + domain_label text not null, + primary key (revision_id, domain_label) + ); + + create table "ClaimsList" ( + revision_id bigserial not null, + creation_timestamp timestamptz not null, + primary key (revision_id) + ); + + alter table "ClaimsEntry" + add constraint FKlugn0q07ayrtar87dqi3vs3c8 + foreign key (revision_id) + references "ClaimsList"; diff --git a/gradle.properties b/gradle.properties index 67a38dd16..9c64a627e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -7,3 +7,11 @@ uploaderMultithreadedUpload= verboseTestOutput=false flowDocsFile= enableDependencyLocking=true + +# Cloud SQL properties + +# A registry environment name (e.g., 'alpha') or a host[:port] string +dbServer= +dbName=postgres +dbUser= +dbPassword= diff --git a/utils.gradle b/utils.gradle new file mode 100644 index 000000000..940edd012 --- /dev/null +++ b/utils.gradle @@ -0,0 +1,33 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +rootProject.ext { + + // Executes an arbitrary shell command in bash and returns all output + // to stdout as a string. This method allows pipes in shell command. + execInBash = { shellCommand, bashWorkingDir -> + return new ByteArrayOutputStream().withStream { os -> + exec { + workingDir bashWorkingDir + commandLine 'bash', '-c', "${shellCommand}" + standardOutput os + }.assertNormalExitValue() + return os + }.toString().trim() + } + + findOptionalProperty = { propertyName -> + return java.util.Optional.ofNullable(project.findProperty(propertyName)) + } +}