mirror of
https://github.com/google/nomulus.git
synced 2025-04-29 19:47:51 +02:00
316 lines
11 KiB
Python
316 lines
11 KiB
Python
# Copyright 2019 The Nomulus Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Google-style presubmits for the Nomulus project.
|
|
|
|
These aren't built in to the static code analysis tools we use (e.g. Checkstyle,
|
|
Error Prone) so we must write them manually.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
from typing import List, Tuple
|
|
import sys
|
|
import textwrap
|
|
import re
|
|
|
|
# We should never analyze any generated files
|
|
UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/", ".gradle/", "/dist/", "karma.conf.js", "polyfills.ts", "test.ts", "/docs/console-endpoints/"}
|
|
# We can't rely on CI to have the Enum package installed so we do this instead.
|
|
FORBIDDEN = 1
|
|
REQUIRED = 2
|
|
|
|
# The list of expected json packages and their licenses.
|
|
# These should be one of the allowed licenses in:
|
|
# config/dependency-license/allowed_licenses.json
|
|
EXPECTED_JS_PACKAGES = [
|
|
'google-closure-library', # Owned by Google, Apache 2.0
|
|
]
|
|
|
|
|
|
class PresubmitCheck:
|
|
|
|
def __init__(self,
|
|
regex,
|
|
included_extensions,
|
|
skipped_patterns,
|
|
regex_type=FORBIDDEN):
|
|
"""Define a presubmit check for a particular set of files,
|
|
|
|
The provided prefix should always or never be included in the files.
|
|
|
|
Args:
|
|
regex: the regular expression to forbid or require
|
|
included_extensions: a tuple of extensions that define which files
|
|
we run over
|
|
skipped_patterns: a set of patterns that will cause any files that
|
|
include them to be skipped
|
|
regex_type: either FORBIDDEN or REQUIRED--whether or not the regex
|
|
must be present or cannot be present.
|
|
"""
|
|
self.regex = regex
|
|
self.included_extensions = included_extensions
|
|
self.skipped_patterns = UNIVERSALLY_SKIPPED_PATTERNS.union(skipped_patterns)
|
|
self.regex_type = regex_type
|
|
|
|
def fails(self, file):
|
|
""" Determine whether or not this file fails the regex check.
|
|
|
|
Args:
|
|
file: the full path of the file to check
|
|
"""
|
|
if not file.endswith(self.included_extensions):
|
|
return False
|
|
for pattern in self.skipped_patterns:
|
|
if pattern in file:
|
|
return False
|
|
with open(file, "r", encoding='utf8') as f:
|
|
file_content = f.read()
|
|
matches = re.match(self.regex, file_content, re.DOTALL)
|
|
if self.regex_type == FORBIDDEN:
|
|
return matches
|
|
return not matches
|
|
|
|
|
|
PRESUBMITS = {
|
|
# License check
|
|
PresubmitCheck(
|
|
r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
|
|
("java", "js", "soy", "sql", "py", "sh", "gradle", "ts"), {
|
|
".git", "/build/", "/generated/", "/generated_tests/",
|
|
"node_modules/", "LoggerConfig.java", "registrar_bin.",
|
|
"registrar_dbg.", "google-java-format-diff.py",
|
|
"nomulus.golden.sql", "soyutils_usegoog.js", "javascript/checks.js"
|
|
}, REQUIRED):
|
|
"File did not include the license header.",
|
|
|
|
# Files must end in a newline
|
|
PresubmitCheck(r".*\n$", ("java", "js", "soy", "sql", "py", "sh", "gradle", "ts"),
|
|
{"node_modules/"}, REQUIRED):
|
|
"Source files must end in a newline.",
|
|
|
|
# System.(out|err).println should only appear in tools/
|
|
PresubmitCheck(
|
|
r".*\bSystem\.(out|err)\.print", "java", {
|
|
"StackdriverDashboardBuilder.java", "/tools/", "/example/",
|
|
"RegistryTestServerMain.java", "TestServerExtension.java",
|
|
"FlowDocumentationTool.java"
|
|
}):
|
|
"System.(out|err).println is only allowed in tools/ packages. Please "
|
|
"use a logger instead.",
|
|
|
|
# PostgreSQLContainer instantiation must specify docker tag
|
|
# TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
|
|
PresubmitCheck(
|
|
r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
|
|
"java", {"DatabaseSnapshotTest.java"}):
|
|
"PostgreSQLContainer instantiation must specify docker tag.",
|
|
|
|
# Various Soy linting checks
|
|
PresubmitCheck(
|
|
r".* (/\*)?\* {?@param ",
|
|
"soy",
|
|
{},
|
|
):
|
|
"In SOY please use the ({@param name: string} /** User name. */) style"
|
|
" parameter passing instead of the ( * @param name User name.) style "
|
|
"parameter pasing.",
|
|
PresubmitCheck(
|
|
r'.*\{[^}]+\w+:\s+"',
|
|
"soy",
|
|
{},
|
|
):
|
|
"Please don't use double-quoted string literals in Soy parameters",
|
|
PresubmitCheck(
|
|
r'.*autoescape\s*=\s*"[^s]',
|
|
"soy",
|
|
{},
|
|
):
|
|
"All soy templates must use strict autoescaping",
|
|
PresubmitCheck(
|
|
r".*noAutoescape",
|
|
"soy",
|
|
{},
|
|
):
|
|
"All soy templates must use strict autoescaping",
|
|
|
|
# various JS linting checks
|
|
PresubmitCheck(
|
|
r".*goog\.base\(",
|
|
"js",
|
|
{"/node_modules/"},
|
|
):
|
|
"Use of goog.base is not allowed.",
|
|
PresubmitCheck(
|
|
r".*goog\.dom\.classes",
|
|
"js",
|
|
{"/node_modules/"},
|
|
):
|
|
"Instead of goog.dom.classes, use goog.dom.classlist which is smaller "
|
|
"and faster.",
|
|
PresubmitCheck(
|
|
r".*goog\.getMsg",
|
|
"js",
|
|
{"/node_modules/"},
|
|
):
|
|
"Put messages in Soy, instead of using goog.getMsg().",
|
|
PresubmitCheck(
|
|
r".*(innerHTML|outerHTML)\s*(=|[+]=)([^=]|$)",
|
|
"js",
|
|
{"/node_modules/", "registrar_bin."},
|
|
):
|
|
"Do not assign directly to the dom. Use goog.dom.setTextContent to set"
|
|
" to plain text, goog.dom.removeChildren to clear, or "
|
|
"soy.renderElement to render anything else",
|
|
PresubmitCheck(
|
|
r".*console\.(log|info|warn|error)",
|
|
"js",
|
|
{"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
|
|
):
|
|
"JavaScript files should not include console logging.",
|
|
}
|
|
|
|
# Note that this regex only works for one kind of Flyway file. If we want to
|
|
# start using "R" and "U" files we'll need to update this script.
|
|
FLYWAY_FILE_RX = re.compile(r'V(\d+)__.*')
|
|
|
|
|
|
def get_seqnum(filename: str, location: str) -> int:
|
|
"""Extracts the sequence number from a filename."""
|
|
m = FLYWAY_FILE_RX.match(filename)
|
|
if m is None:
|
|
raise ValueError('Illegal Flyway filename: %s in %s' % (filename, location))
|
|
return int(m.group(1))
|
|
|
|
|
|
def files_by_seqnum(files: List[str], location: str) -> List[Tuple[int, str]]:
|
|
"""Returns the list of seqnum, filename sorted by sequence number."""
|
|
return [(get_seqnum(filename, location), filename) for filename in files]
|
|
|
|
|
|
def has_valid_order(indexed_files: List[Tuple[int, str]], location: str) -> bool:
|
|
"""Verify that sequence numbers are in order without gaps or duplicates.
|
|
|
|
Args:
|
|
files: List of seqnum, filename for a list of Flyway files.
|
|
location: Where the list of files came from (for error reporting).
|
|
|
|
Returns:
|
|
True if the file list is valid.
|
|
"""
|
|
last_index = 0
|
|
valid = True
|
|
for seqnum, filename in indexed_files:
|
|
if seqnum == last_index:
|
|
print('duplicate Flyway file sequence number found in %s: %s' %
|
|
(location, filename))
|
|
valid = False
|
|
elif seqnum < last_index:
|
|
print('File %s in %s is out of order.' % (filename, location))
|
|
valid = False
|
|
elif seqnum != last_index + 1:
|
|
print('Missing Flyway sequence number %d in %s. Next file is %s' %
|
|
(last_index + 1, location, filename))
|
|
valid = False
|
|
last_index = seqnum
|
|
return valid
|
|
|
|
|
|
def verify_flyway_index():
|
|
"""Verifies that the Flyway index file is in sync with the directory."""
|
|
success = True
|
|
|
|
# Sort the files in the Flyway directory by their sequence number.
|
|
files = sorted(
|
|
files_by_seqnum(os.listdir('db/src/main/resources/sql/flyway'),
|
|
'Flyway directory'))
|
|
|
|
# Make sure that there are no gaps and no duplicate sequence numbers in the
|
|
# files themselves.
|
|
if not has_valid_order(files, 'Flyway directory'):
|
|
success = False
|
|
|
|
# Remove the sequence numbers and compare against the index file contents.
|
|
files = [filename[1] for filename in sorted(files)]
|
|
with open('db/src/main/resources/sql/flyway.txt', encoding='utf8') as index:
|
|
indexed_files = index.read().splitlines()
|
|
if files != indexed_files:
|
|
unindexed = set(files) - set(indexed_files)
|
|
if unindexed:
|
|
print('The following Flyway files are not in flyway.txt: %s' % unindexed)
|
|
|
|
nonexistent = set(indexed_files) - set(files)
|
|
if nonexistent:
|
|
print('The following files are in flyway.txt but not in the Flyway '
|
|
'directory: %s' % nonexistent)
|
|
|
|
# Do an ordering check on the index file (ignore the result, we're failing
|
|
# anyway).
|
|
has_valid_order(files_by_seqnum(indexed_files, 'flyway.txt'), 'flyway.txt')
|
|
success = False
|
|
|
|
if not success:
|
|
print('Please fix any conflicts and run "./nom_build :db:generateFlywayIndex"')
|
|
|
|
return not success
|
|
|
|
|
|
def verify_javascript_deps():
|
|
"""Verifies that we haven't introduced any new javascript dependencies."""
|
|
with open('package.json') as f:
|
|
package = json.load(f)
|
|
|
|
deps = list(package['dependencies'].keys())
|
|
if deps != EXPECTED_JS_PACKAGES:
|
|
print('Unexpected javascript dependencies. Was expecting '
|
|
'%s, got %s.' % (EXPECTED_JS_PACKAGES, deps))
|
|
print(textwrap.dedent("""
|
|
* If the new dependencies are intentional, please verify that the
|
|
* license is one of the allowed licenses (see
|
|
* config/dependency-license/allowed_licenses.json) and add an entry
|
|
* for the package (with the license in a comment) to the
|
|
* EXPECTED_JS_PACKAGES variable in config/presubmits.py.
|
|
"""))
|
|
return True
|
|
return False
|
|
|
|
|
|
def get_files():
|
|
for root, dirnames, filenames in os.walk("."):
|
|
for filename in filenames:
|
|
yield os.path.join(root, filename)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print('python version is %s' % sys.version)
|
|
failed = False
|
|
for file in get_files():
|
|
error_messages = []
|
|
for presubmit, error_message in PRESUBMITS.items():
|
|
if presubmit.fails(file):
|
|
error_messages.append(error_message)
|
|
|
|
if error_messages:
|
|
failed = True
|
|
print("%s had errors: \n %s" % (file, "\n ".join(error_messages)))
|
|
|
|
# And now for something completely different: check to see if the Flyway
|
|
# index is up-to-date. It's quicker to do it here than in the unit tests:
|
|
# when we put it here it fails fast before all of the tests are run.
|
|
failed |= verify_flyway_index()
|
|
|
|
# Make sure we haven't introduced any javascript dependencies.
|
|
failed |= verify_javascript_deps()
|
|
|
|
if failed:
|
|
sys.exit(1)
|