Add a generate_sql_schema command (#230)

* Add a generate_schema  command

Add a generate_schema command to nomulus tool and add the necessary
instrumentation to EppResource and DomainBase to allow us to generate a
proof-of-concept schema for DomainBase.

* Added forgotten command description

* Revert "Added forgotten command description"

This reverts commit 09326cb8ac.
(checked in the wrong file)

* Added fixes requested during review

* Add a todo to start postgresql container

Add a todo to start a postgresql container from generate_sql_command.

config/dependency-license/allowed_licenses.json

@@ -75,6 +75,9 @@
     {
       "moduleLicense": "BSD style"
     },
+    {
+      "moduleLicense": "BSD-2-Clause"
+    },
     {
       "moduleLicense": "New BSD License"
     },
@@ -84,6 +87,9 @@
     {
       "moduleLicense": "The BSD License"
     },
+    {
+      "moduleLicense": "The PostgreSQL License"
+    },
     {
       "moduleLicense": "CC0 1.0 Universal License"
     },
@@ -105,6 +111,9 @@
     {
       "moduleLicense": "\\n                Dual license consisting of the CDDL v1.1 and GPL v2\\n            "
     },
+    {
+      "moduleLicense": "Eclipse Distribution License v. 1.0"
+    },
     {
       "moduleLicense": "Eclipse Public License - Version 1.0"
     },
@@ -135,6 +144,9 @@
     {
       "moduleLicense": "GNU Lesser Public License"
     },
+    {
+      "moduleLicense": "GNU Library General Public License v2.1 or later"
+    },
     {
       "moduleLicense": "The Go license"
     },

core/build.gradle

@@ -53,15 +53,19 @@ def fragileTestPatterns = [
     // Test Datastore inexplicably aborts transaction.
     "google/registry/model/tmch/ClaimsListShardTest.*",
     // Creates large object (64MBytes), occasionally throws OOM error.
-    "google/registry/model/server/KmsSecretRevisionTest.*"
+    "google/registry/model/server/KmsSecretRevisionTest.*",
+    "google/registry/tools/GenerateSqlSchemaCommandTest.*",
+    "google/registry/webdriver/*",
 ]
 
 // Tests that fail when running Gradle in a docker container, e. g. when
 // building the release artifacts in Google Cloud Build.
 def dockerIncompatibleTestPatterns = [
     // The webdriver tests start headless Chrome in a Docker container,
-    // resulting in Docker-in-Docker complications.
+    // resulting in Docker-in-Docker complications.  Likewise,
+    // GenerateSqlSchemaCommandTest launches postgresql in a docker container.
     "google/registry/webdriver/*",
+    "google/registry/tools/GenerateSqlSchemaCommandTest.*",
     // PathParameterTest includes tests which validate that file permissions are
     // respected. However when running in Docker the user is root by default, so
     // every file is read/write-able. There is no way to exclude specific test
@@ -214,13 +218,16 @@ dependencies {
   compile deps['org.bouncycastle:bcpg-jdk15on']
   testCompile deps['org.bouncycastle:bcpkix-jdk15on']
   compile deps['org.bouncycastle:bcprov-jdk15on']
+  compile deps['org.hibernate:hibernate-core']
   compile deps['org.joda:joda-money']
   compile deps['org.json:json']
   testCompile deps['org.mortbay.jetty:jetty']
+  runtimeOnly deps['org.postgresql:postgresql']
   testCompile deps['org.seleniumhq.selenium:selenium-api']
   testCompile deps['org.seleniumhq.selenium:selenium-chrome-driver']
   testCompile deps['org.seleniumhq.selenium:selenium-java']
   testCompile deps['org.seleniumhq.selenium:selenium-remote-driver']
+  testCompile deps['org.testcontainers:postgresql']
   testCompile deps['org.testcontainers:selenium']
   compile deps['xerces:xmlParserAPIs']
   compile deps['xpp3:xpp3']

core/src/main/java/google/registry/model/EppResource.java

@@ -50,10 +50,13 @@ import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import javax.persistence.MappedSuperclass;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /** An EPP entity object (i.e. a domain, contact, or host). */
+@MappedSuperclass
 public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /**
@@ -62,8 +65,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>This is in the (\w|_){1,80}-\w{1,8} format specified by RFC 5730 for roidType.
    * @see <a href="https://tools.ietf.org/html/rfc5730">RFC 5730</a>
    */
-  @Id
+  @Id @javax.persistence.Id String repoId;
-  String repoId;
 
   /** The ID of the registrar that is currently sponsoring this resource. */
   @Index
@@ -85,8 +87,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   // Map the method to XML, not the field, because if we map the field (with an adaptor class) it
   // will never be omitted from the xml even if the timestamp inside creationTime is null and we
   // return null from the adaptor. (Instead it gets written as an empty tag.)
-  @Index
+  @Index @Transient CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
-  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -115,7 +116,7 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
   DateTime lastEppUpdateTime;
 
   /** Status values associated with this resource. */
-  Set<StatusValue> status;
+  @Transient Set<StatusValue> status;
 
   /**
    * Sorted map of {@link DateTime} keys (modified time) to {@link CommitLogManifest} entries.

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -69,6 +69,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.function.Predicate;
 import javax.annotation.Nullable;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -83,6 +84,8 @@ import org.joda.time.Interval;
  */
 @ReportedOn
 @Entity
+@javax.persistence.Entity
+@javax.persistence.Table(name = "domain")
 @ExternalMessagingName("domain")
 public class DomainBase extends EppResource
     implements ForeignKeyedEppResource, ResourceWithTransferData {
@@ -115,18 +118,17 @@ public class DomainBase extends EppResource
   String tld;
 
   /** References to hosts that are the nameservers for the domain. */
-  @Index
+  @Index @Transient Set<Key<HostResource>> nsHosts;
-  Set<Key<HostResource>> nsHosts;
 
   /**
    * The union of the contacts visible via {@link #getContacts} and {@link #getRegistrant}.
    *
    * <p>These are stored in one field so that we can query across all contacts at once.
    */
-  Set<DesignatedContact> allContacts;
+  @Transient Set<DesignatedContact> allContacts;
 
   /** Authorization info (aka transfer secret) of the domain. */
-  DomainAuthInfo authInfo;
+  @Transient DomainAuthInfo authInfo;
 
   /**
    * Data used to construct DS records for this domain.
@@ -134,14 +136,13 @@ public class DomainBase extends EppResource
    * <p>This is {@literal @}XmlTransient because it needs to be returned under the "extension" tag
    * of an info response rather than inside the "infData" tag.
    */
-  Set<DelegationSignerData> dsData;
+  @Transient Set<DelegationSignerData> dsData;
 
   /**
    * The claims notice supplied when this application or domain was created, if there was one. It's
    * {@literal @}XmlTransient because it's not returned in an info response.
    */
-  @IgnoreSave(IfNull.class)
+  @IgnoreSave(IfNull.class) @Transient LaunchNotice launchNotice;
-  LaunchNotice launchNotice;
 
   /**
    * Name of first IDN table associated with TLD that matched the characters in this domain label.
@@ -152,7 +153,7 @@ public class DomainBase extends EppResource
   String idnTableName;
 
   /** Fully qualified host names of this domain's active subordinate hosts. */
-  Set<String> subordinateHosts;
+  @Transient Set<String> subordinateHosts;
 
   /** When this domain's registration will expire. */
   DateTime registrationExpirationTime;
@@ -187,7 +188,7 @@ public class DomainBase extends EppResource
   Key<PollMessage.Autorenew> autorenewPollMessage;
 
   /** The unexpired grace periods for this domain (some of which may not be active yet). */
-  Set<GracePeriod> gracePeriods;
+  @Transient Set<GracePeriod> gracePeriods;
 
   /**
    * The id of the signed mark that was used to create this domain in sunrise.
@@ -198,7 +199,7 @@ public class DomainBase extends EppResource
   String smdId;
 
   /** Data about any pending or past transfers on this domain. */
-  TransferData transferData;
+  @Transient TransferData transferData;
 
   /**
    * The time that this resource was last transferred.

core/src/main/java/google/registry/tools/GenerateSqlSchemaCommand.java

@@ -0,0 +1,79 @@
+// Copyright 2019 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import google.registry.model.domain.DomainBase;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+import org.hibernate.boot.MetadataSources;
+import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
+import org.hibernate.tool.hbm2ddl.SchemaExport;
+import org.hibernate.tool.schema.TargetType;
+
+/**
+ * Generates a schema for JPA annotated classes using hibernate.
+ *
+ * <p>Note that this isn't complete yet, as all of the persistent classes have not yet been
+ * converted. After converting a class, a call to "addAnnotatedClass()" for the new class must be
+ * added to the code below.
+ */
+@Parameters(separators = " =", commandDescription = "Generate postgresql schema.")
+public class GenerateSqlSchemaCommand implements Command {
+
+  public static final int POSTGRESQL_PORT = 5432;
+
+  @Parameter(
+      names = {"-o", "--out-file"},
+      description = "")
+  String outFile;
+
+  @Parameter(
+      names = {"-a", "--db-host"},
+      description = "Database host name.")
+  String databaseHost;
+
+  @Parameter(
+      names = {"-p", "--db-port"},
+      description = "Database port number.  This defaults to the postgresql default port.")
+  int databasePort = POSTGRESQL_PORT;
+
+  @Override
+  public void run() {
+    // TODO(mmuller): Optionally (and perhaps by default) start a postgresql instance container
+    // rather than relying on the user to have one to connect to.
+    Map<String, String> settings = new HashMap<>();
+    settings.put("hibernate.dialect", "org.hibernate.dialect.PostgreSQL9Dialect");
+    settings.put(
+        "hibernate.connection.url",
+        "jdbc:postgresql://" + databaseHost + ":" + databasePort + "/postgres?useSSL=false");
+    settings.put("hibernate.connection.username", "postgres");
+    settings.put("hibernate.connection.password", "domain-registry");
+    settings.put("hibernate.hbm2ddl.auto", "none");
+    settings.put("show_sql", "true");
+
+    MetadataSources metadata =
+        new MetadataSources(new StandardServiceRegistryBuilder().applySettings(settings).build());
+    metadata.addAnnotatedClass(DomainBase.class);
+    SchemaExport schemaExport = new SchemaExport();
+    schemaExport.setHaltOnError(true);
+    schemaExport.setFormat(true);
+    schemaExport.setDelimiter(";");
+    schemaExport.setOutputFile(outFile);
+    schemaExport.createOnly(EnumSet.of(TargetType.SCRIPT), metadata.buildMetadata());
+  }
+}

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -61,6 +61,7 @@ public final class RegistryTool {
           .put("generate_dns_report", GenerateDnsReportCommand.class)
           .put("generate_escrow_deposit", GenerateEscrowDepositCommand.class)
           .put("generate_lordn", GenerateLordnCommand.class)
+          .put("generate_sql_schema", GenerateSqlSchemaCommand.class)
           .put("generate_zone_files", GenerateZoneFilesCommand.class)
           .put("get_allocation_token", GetAllocationTokenCommand.class)
           .put("get_claims_list", GetClaimsListCommand.class)

core/src/test/java/google/registry/tools/GenerateSqlSchemaCommandTest.java

@@ -0,0 +1,64 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package google.registry.tools;
+
+// import static org.mockito.Mockito.mock;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.io.File;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.testcontainers.containers.PostgreSQLContainer;
+
+
+@RunWith(JUnit4.class)
+public class GenerateSqlSchemaCommandTest extends CommandTestCase<GenerateSqlSchemaCommand> {
+
+  private String containerHostName;
+  private int containerPort;
+
+  @Rule public TemporaryFolder tmp = new TemporaryFolder();
+
+  @Rule public PostgreSQLContainer postgres =
+      new PostgreSQLContainer()
+          .withDatabaseName("postgres")
+          .withUsername("postgres")
+          .withPassword("domain-registry");
+
+  public GenerateSqlSchemaCommandTest() {}
+
+  @Before
+  public void setUp() {
+    containerHostName = postgres.getContainerIpAddress();
+    containerPort = postgres.getMappedPort(GenerateSqlSchemaCommand.POSTGRESQL_PORT);
+  }
+
+  @Test
+  public void testSchemaGeneration() throws Exception {
+    runCommand(
+        "--out-file=" + tmp.getRoot() + File.separatorChar + "schema.sql",
+        "--db-host=" + containerHostName,
+        "--db-port=" + containerPort);
+
+    // We're just interested in verifying that there is a schema file generated, we don't do any
+    // checks on the contents, this would make the test too brittle and serves no real purpose.
+    // TODO: try running the schema against the test database.
+    assertThat(new File(tmp.getRoot(), "schema.sql").exists()).isTrue();
+  }
+}

dependencies.gradle

@@ -126,6 +126,7 @@ ext {
       'org.seleniumhq.selenium:selenium-java:3.141.59',
       'org.seleniumhq.selenium:selenium-remote-driver:3.141.59',
       'org.testcontainers:selenium:1.10.7',
+      'org.testcontainers:postgresql:1.8.3',
       'org.yaml:snakeyaml:1.17',
       'xerces:xmlParserAPIs:2.6.2',
       'xpp3:xpp3:1.1.4c'