From 9de287378b1691fb10fac56cd9ffc2bc6a068725 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Mon, 1 Aug 2016 15:34:13 -0700 Subject: [PATCH] Create zip_file() rule This doesn't work quite the same way as Fileset(). But it should be able to serve as a decent replacement. The important part of this design is that zips can depend on other zips. Therefore definitions don't have to be monolithic. This will be important when migrating the Domain Registry codebase, because our Fileset() definitions are dispersed across many BUILD files. So we'll be able to migrate to zip_file() with the least amount of intrusiveness. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=129034885 --- java/google/registry/builddefs/BUILD | 4 + java/google/registry/builddefs/defs.bzl | 26 ++++ java/google/registry/builddefs/zip_file.bzl | 113 ++++++++++++++++++ javatests/google/registry/builddefs/BUILD | 92 ++++++++++++++ javatests/google/registry/builddefs/hello.txt | 1 + .../registry/builddefs/override/hello.txt | 1 + javatests/google/registry/builddefs/world.txt | 1 + .../registry/builddefs/zip_contents_test.bzl | 61 ++++++++++ 8 files changed, 299 insertions(+) create mode 100644 java/google/registry/builddefs/BUILD create mode 100644 java/google/registry/builddefs/defs.bzl create mode 100644 java/google/registry/builddefs/zip_file.bzl create mode 100644 javatests/google/registry/builddefs/BUILD create mode 100644 javatests/google/registry/builddefs/hello.txt create mode 100644 javatests/google/registry/builddefs/override/hello.txt create mode 100644 javatests/google/registry/builddefs/world.txt create mode 100644 javatests/google/registry/builddefs/zip_contents_test.bzl diff --git a/java/google/registry/builddefs/BUILD b/java/google/registry/builddefs/BUILD new file mode 100644 index 000000000..0ac9f966a --- /dev/null +++ b/java/google/registry/builddefs/BUILD @@ -0,0 +1,4 @@ +package(default_visibility = ["//java/google/registry:registry_project"]) + +licenses(["notice"]) # Apache 2.0 + diff --git a/java/google/registry/builddefs/defs.bzl b/java/google/registry/builddefs/defs.bzl new file mode 100644 index 000000000..243fb06b0 --- /dev/null +++ b/java/google/registry/builddefs/defs.bzl @@ -0,0 +1,26 @@ +# Copyright 2016 The Domain Registry Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common routines for Domain Registry build rules.""" + +ZIPPER = "@bazel_tools//tools/zip:zipper" + +_OUTPUT_DIRS = ("bazel-out/", "bazel-genfiles/") + +def runpath(f): + """Figures out the proper runfiles path for a file.""" + for prefix in _OUTPUT_DIRS: + if f.path.startswith(prefix): + return f.short_path + return f.path diff --git a/java/google/registry/builddefs/zip_file.bzl b/java/google/registry/builddefs/zip_file.bzl new file mode 100644 index 000000000..53f9e7e6a --- /dev/null +++ b/java/google/registry/builddefs/zip_file.bzl @@ -0,0 +1,113 @@ +# Copyright 2016 The Domain Registry Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +load('//java/google/registry/builddefs:defs.bzl', 'ZIPPER', 'runpath') + +def _impl(ctx): + """Implementation of zip_file() rule.""" + for s, d in ctx.attr.mappings.items(): + if (s.startswith('/') or s.endswith('/') or + d.startswith('/') or d.endswith('/')): + fail('mappings should not begin or end with slash') + mapped = _map_sources(ctx.files.srcs, ctx.attr.mappings) + cmd = [ + 'set -e', + 'repo="$(pwd)"', + 'zipper="${repo}/%s"' % ctx.file._zipper.path, + 'archive="${repo}/%s"' % ctx.outputs.out.path, + 'tmp="$(mktemp -d "${TMPDIR:-/tmp}/zip_file.XXXXXXXXXX")"', + 'cd "${tmp}"', + ] + cmd += ['"${zipper}" x "${repo}/%s"' % dep.zip_file.path + for dep in ctx.attr.deps] + cmd += ['mkdir -p "${tmp}/%s"' % zip_path + for zip_path in set( + [zip_path[:zip_path.rindex('/')] + for _, zip_path in mapped if '/' in zip_path])] + cmd += ['ln -sf "${repo}/%s" "${tmp}/%s"' % (path, zip_path) + for path, zip_path in mapped] + cmd += [ + ('find . | sed 1d | cut -c 3- | LC_ALL=C sort' + + ' | xargs "${zipper}" cC "${archive}"'), + 'cd "${repo}"', + 'rm -rf "${tmp}"', + ] + inputs = [ctx.file._zipper] + inputs += [dep.zip_file for dep in ctx.attr.deps] + inputs += ctx.files.srcs + ctx.action( + inputs=inputs, + outputs=[ctx.outputs.out], + command='\n'.join(cmd), + mnemonic='zip', + progress_message='Creating zip with %d inputs %s' % ( + len(inputs), ctx.label)) + return struct(files=set([ctx.outputs.out]), + zip_file=ctx.outputs.out) + +def _map_sources(srcs, mappings): + """Calculates paths in zip file for srcs.""" + # order mappings with more path components first + mappings = sorted([(-len(source.split('/')), source, dest) + for source, dest in mappings.items()]) + # get rid of the integer part of tuple used for sorting + mappings = [(source, dest) for _, source, dest in mappings] + mappings_indexes = range(len(mappings)) + used = {i: False for i in mappings_indexes} + mapped = [] + for file_ in srcs: + short_path = runpath(file_) + zip_path = None + for i in mappings_indexes: + source = mappings[i][0] + dest = mappings[i][1] + if not source: + if dest: + zip_path = dest + '/' + short_path + else: + zip_path = short_path + elif source == short_path: + if dest: + zip_path = dest + else: + zip_path = short_path + elif short_path.startswith(source + '/'): + if dest: + zip_path = dest + short_path[len(source):] + else: + zip_path = short_path[len(source) + 1:] + else: + continue + used[i] = True + break + if not zip_path: + fail('no mapping matched: ' + short_path) + mapped += [(file_.path, zip_path)] + for i in mappings_indexes: + if not used[i]: + fail('superfluous mapping: "%s" -> "%s"' % mappings[i]) + return mapped + +zip_file = rule( + implementation=_impl, + output_to_genfiles = True, + attrs={ + 'out': attr.output(mandatory=True), + 'srcs': attr.label_list(allow_files=True), + 'deps': attr.label_list(providers=['zip_file']), + 'mappings': attr.string_dict(), + '_zipper': attr.label(default=Label(ZIPPER), single_file=True), + }) diff --git a/javatests/google/registry/builddefs/BUILD b/javatests/google/registry/builddefs/BUILD new file mode 100644 index 000000000..00c4c9a2c --- /dev/null +++ b/javatests/google/registry/builddefs/BUILD @@ -0,0 +1,92 @@ +package( + default_testonly = 1, + default_visibility = ["//java/google/registry:registry_project"], +) + +licenses(["notice"]) # Apache 2.0 + +load("//java/google/registry/builddefs:zip_file.bzl", "zip_file") +load("//javatests/google/registry/builddefs:zip_contents_test.bzl", "zip_contents_test") + + +genrule( + name = "generated", + outs = ["generated.txt"], + cmd = "echo generated >$@", +) + +zip_file( + name = "basic", + srcs = [ + "generated.txt", + "hello.txt", + "world.txt", + ], + out = "basic.zip", + mappings = {"": ""}, +) + +zip_contents_test( + name = "zip_emptyMapping_leavesShortPathsInTact", + src = "basic.zip", + contents = { + "javatests/google/registry/builddefs/generated.txt": "generated", + "javatests/google/registry/builddefs/hello.txt": "hello", + "javatests/google/registry/builddefs/world.txt": "world", + }, +) + +zip_file( + name = "stripped", + srcs = ["hello.txt"], + out = "stripped.zip", + mappings = {"javatests/google/registry/builddefs": ""}, +) + +zip_contents_test( + name = "zip_prefixRemoval_works", + src = "stripped.zip", + contents = {"hello.txt": "hello"}, +) + +zip_file( + name = "repath", + srcs = [ + "generated.txt", + "hello.txt", + "world.txt", + ], + out = "repath.zip", + mappings = { + "javatests/google/registry/builddefs": "a/b/c", + "javatests/google/registry/builddefs/generated.txt": "x/y/z/generated.txt", + }, +) + +zip_contents_test( + name = "zip_pathReplacement_works", + src = "repath.zip", + contents = { + "a/b/c/hello.txt": "hello", + "a/b/c/world.txt": "world", + "x/y/z/generated.txt": "generated", + }, +) + +zip_file( + name = "overridden", + srcs = ["override/hello.txt"], + out = "overridden.zip", + mappings = {"javatests/google/registry/builddefs/override": "a/b/c"}, + deps = [":repath"], +) + +zip_contents_test( + name = "zip_fileWithSameMappingAsDependentRule_prefersMyMapping", + src = "overridden.zip", + contents = { + "a/b/c/hello.txt": "OMG IM AN OVERRIDE", + "a/b/c/world.txt": "world", + "x/y/z/generated.txt": "generated", + }, +) diff --git a/javatests/google/registry/builddefs/hello.txt b/javatests/google/registry/builddefs/hello.txt new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/javatests/google/registry/builddefs/hello.txt @@ -0,0 +1 @@ +hello diff --git a/javatests/google/registry/builddefs/override/hello.txt b/javatests/google/registry/builddefs/override/hello.txt new file mode 100644 index 000000000..ef038d6e8 --- /dev/null +++ b/javatests/google/registry/builddefs/override/hello.txt @@ -0,0 +1 @@ +OMG IM AN OVERRIDE diff --git a/javatests/google/registry/builddefs/world.txt b/javatests/google/registry/builddefs/world.txt new file mode 100644 index 000000000..cc628ccd1 --- /dev/null +++ b/javatests/google/registry/builddefs/world.txt @@ -0,0 +1 @@ +world diff --git a/javatests/google/registry/builddefs/zip_contents_test.bzl b/javatests/google/registry/builddefs/zip_contents_test.bzl new file mode 100644 index 000000000..661f1dc40 --- /dev/null +++ b/javatests/google/registry/builddefs/zip_contents_test.bzl @@ -0,0 +1,61 @@ +# Copyright 2016 The Domain Registry Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build rule for unit testing the zip_file() rule.""" + +load('//java/google/registry/builddefs:defs.bzl', 'ZIPPER', 'runpath') + +def _impl(ctx): + """Implementation of zip_contents_test() rule.""" + cmd = [ + 'set -e', + 'repo="$(pwd)"', + 'zipper="${repo}/%s"' % runpath(ctx.file._zipper), + 'archive="${repo}/%s"' % runpath(ctx.file.src), + ('listing="$("${zipper}" v "${archive}"' + + ' | grep -v ^d | awk \'{print $3}\' | LC_ALL=C sort)"'), + 'if [[ "${listing}" != "%s" ]]; then' % ( + '\n'.join(ctx.attr.contents.keys())), + ' echo "archive had different file listing:"', + ' "${zipper}" v "${archive}" | grep -v ^d', + ' exit 1', + 'fi', + 'tmp="$(mktemp -d "${TMPDIR:-/tmp}/zip_contents_test.XXXXXXXXXX")"', + 'cd "${tmp}"', + '"${zipper}" x "${archive}"', + ] + for path, data in ctx.attr.contents.items(): + cmd += [ + 'if [[ "$(cat "%s")" != "%s" ]]; then' % (path, data), + ' echo "%s had different contents:"' % path, + ' cat "%s"' % path, + ' exit 1', + 'fi', + ] + cmd += ['cd "${repo}"', + 'rm -rf "${tmp}"'] + ctx.file_action( + output=ctx.outputs.executable, + content='\n'.join(cmd), + executable=True) + return struct(runfiles=ctx.runfiles([ctx.file.src, ctx.file._zipper])) + +zip_contents_test = rule( + implementation=_impl, + test=True, + attrs={ + 'src': attr.label(allow_single_file=True), + 'contents': attr.string_dict(), + '_zipper': attr.label(default=Label(ZIPPER), single_file=True), + })