From a03d10ce22bb84921fc76eeecdc5273fe12f5669 Mon Sep 17 00:00:00 2001 From: guyben Date: Tue, 12 Mar 2019 11:10:30 -0700 Subject: [PATCH] Add ability to save report to local folder Updated the plugin to receive the "protocol"-like tag in the destination, so that you can choose whether to upload to GCS or just save it locally. Possibly we might expand this in the future, but for now the goal was to allow saving our "internal" builds locally until we find a secure way to store AND BROWSE them remotely. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=238055136 --- .travis.yml | 9 +- gradle/build.gradle | 16 +-- .../gradle/plugin/CoverPageGenerator.java | 19 +-- .../gradle/plugin/FilesWithEntryPoint.java | 59 ++++++++ .../gradle/plugin/GcsPluginUtils.java | 28 ++-- .../registry/gradle/plugin/ProjectData.java | 43 +----- ...eportUploader.java => ReportUploader.java} | 136 +++++++++++++----- ...rPlugin.java => ReportUploaderPlugin.java} | 14 +- .../gradle/plugin/CoverPageGeneratorTest.java | 21 ++- .../gradle/plugin/GcsPluginUtilsTest.java | 25 ++-- gradle/gradle.properties | 6 +- 11 files changed, 229 insertions(+), 147 deletions(-) create mode 100644 gradle/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java rename gradle/buildSrc/src/main/java/google/registry/gradle/plugin/{GcsReportUploader.java => ReportUploader.java} (64%) rename gradle/buildSrc/src/main/java/google/registry/gradle/plugin/{GcsReportUploaderPlugin.java => ReportUploaderPlugin.java} (72%) diff --git a/.travis.yml b/.travis.yml index 12b1b420f..54a9f6cb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,9 @@ # "CREDZ" and "REPORT_GCS_BUCKET" environment variables in your travis # repository. # -# The report bucket id should just be the bucket id without any scheme or path -# (e.g. "my-bucket-name", not "gcs://my-bucket-name"). You'll want to select -# "Display value in build log", otherwise travis will hide the bucket name in -# the URL that is displayed. +# The report destination can be any GCS path (e.g. "gcs://bucket-name/path"). +# You'll want to select "Display value in build log", otherwise travis will +# hide the bucket name in the URL that is displayed. # # The CREDZ variable should be the contents of a json credentials file for # a service account with write access to the bucket, escaped for bash shell @@ -56,4 +55,4 @@ env: # output, instead of the default 10. # See notes on the CREDZ and REPORT_GCS_BUCKET environment variable in the # comments at the top of the file. -script: cd gradle && echo "$CREDZ" >credz.json && chmod 755 ./gradlew && travis_wait 45 ./gradlew build --continue -P gcsBucket="$REPORT_GCS_BUCKET" -P gcsCredentialsFile=credz.json -P gcsMultithreadedUpload=yes +script: cd gradle && echo "$CREDZ" >credz.json && chmod 755 ./gradlew && travis_wait 45 ./gradlew build --continue -P uploaderDestination="$REPORT_GCS_DESTINATION" -P uploaderCredentialsFile=credz.json -P uploaderMultithreadedUpload=yes diff --git a/gradle/build.gradle b/gradle/build.gradle index b7a8e8d36..0227127cc 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -23,14 +23,14 @@ plugins { id "com.moowork.node" version "1.2.0" } -apply plugin: google.registry.gradle.plugin.GcsReportUploaderPlugin +apply plugin: google.registry.gradle.plugin.ReportUploaderPlugin -gcsReportUploader { - // Set the bucket here to upload build results to a GCS bucket. - // e.g. -P gcsBucket=domain-registry-alpha-build-result-test +reportUploader { + // Set the location where we want to upload the build results. + // e.g. -P uploaderDestination=gcs://domain-registry-alpha-build-result-test // - // If no bucket it set - the uploading will be skipped. - bucket = gcsBucket + // If not set - the upload will be skipped + destination = uploaderDestination // The location of the file containing the OAuth2 Google Cloud credentials. // @@ -39,11 +39,11 @@ gcsReportUploader { // supported by the Cloud SDK. // // If no file is given - the default credentials are used. - credentialsFile = gcsCredentialsFile + credentialsFile = uploaderCredentialsFile // If set to 'yes', each file will be uploaded to GCS in a separate thread. // This is MUCH faster. - multithreadedUpload = gcsMultithreadedUpload + multithreadedUpload = uploaderMultithreadedUpload } apply from: 'dependencies.gradle' diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java index f71c60607..b29cf794c 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java @@ -46,6 +46,8 @@ final class CoverPageGenerator { /** List of all resource files that will be uploaded as-is. */ private static final ImmutableSet STATIC_RESOURCE_FILES = ImmutableSet.of(Paths.get("css", "style.css")); + /** Name of the entry-point file that will be created. */ + private static final Path ENTRY_POINT = Paths.get("index.html"); private final ProjectData projectData; private final ImmutableSetMultimap tasksByState; @@ -63,30 +65,19 @@ final class CoverPageGenerator { projectData.tasks().stream().collect(toImmutableSetMultimap(TaskData::state, task -> task)); } - /** - * The (relative) entry point for the cover page. - * - *

A file with this relative path is guaranteed to be returned from {@link #getFilesToUpload}, - * and a browser pointing to this page will have access to all the data generated by {@link - * #getFilesToUpload}. - */ - Path getEntryPoint() { - return Paths.get("index.html"); - } - /** * Returns all the files that need uploading for the cover page to work. * *

This includes all the report files as well, to make sure that the link works. */ - ImmutableMap> getFilesToUpload() { + FilesWithEntryPoint getFilesToUpload() { ImmutableMap.Builder> builder = new ImmutableMap.Builder<>(); // Add all the static resource pages STATIC_RESOURCE_FILES.stream().forEach(file -> builder.put(file, resourceLoader(file))); // Create the cover page // Note that the ByteArraySupplier here is lazy - the createCoverPage function is only called // when the resulting Supplier's get function is called. - builder.put(getEntryPoint(), toByteArraySupplier(this::createCoverPage)); + builder.put(ENTRY_POINT, toByteArraySupplier(this::createCoverPage)); // Add all the files from the tasks tasksByState.values().stream() .flatMap(task -> task.reports().values().stream()) @@ -96,7 +87,7 @@ final class CoverPageGenerator { .filter(task -> task.log().isPresent()) .forEach(task -> builder.put(getLogPath(task), task.log().get())); - return builder.build(); + return FilesWithEntryPoint.create(builder.build(), ENTRY_POINT); } /** Renders the cover page. */ diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java new file mode 100644 index 000000000..f8f098b38 --- /dev/null +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java @@ -0,0 +1,59 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.gradle.plugin; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import java.nio.file.Path; +import java.util.function.Supplier; + +/** + * Holds a set of files with a browser-friendly entry point to those files. + * + *

The file data is lazily generated. + * + *

If there is at least one file, it's guaranteed that the entry point is one of these files. + */ +@AutoValue +abstract class FilesWithEntryPoint { + + /** + * All files that are part of this report, keyed from their path to a supplier of their content. + * + *

The reason we use a supplier instead of loading the content is in case the content is very + * large... + * + *

Also, no point in doing IO before we need it! + */ + abstract ImmutableMap> files(); + + /** + * The file that gives access (links...) to all the data in the report. + * + *

Guaranteed to be a key in {@link #files} if and only if files isn't empty. + */ + abstract Path entryPoint(); + + static FilesWithEntryPoint create(ImmutableMap> files, Path entryPoint) { + checkArgument(files.isEmpty() || files.containsKey(entryPoint)); + return new AutoValue_FilesWithEntryPoint(files, entryPoint); + } + + static FilesWithEntryPoint createSingleFile(Path path, Supplier data) { + return create(ImmutableMap.of(path, data), path); + } +} diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java index 99fe3c555..afb47b9a5 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; import com.google.common.io.Files; import com.google.common.io.Resources; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -123,10 +122,10 @@ final class GcsPluginUtils { } /** - * Reads all the files generated by a Report into a ReportFiles object. + * Reads all the files generated by a Report into a FilesWithEntryPoint object. * - *

Every ReportFiles must have a single link "entry point" that gives users access to all the - * files. If the report generated just one file - we will just link to that file. + *

Every FilesWithEntryPoint must have a single link "entry point" that gives users access to + * all the files. If the report generated just one file - we will just link to that file. * *

However, if the report generated more than one file - the only thing we can safely do is to * zip all the files and link to the zip file. @@ -144,20 +143,21 @@ final class GcsPluginUtils { * directory, and (c) {@code entryPointHint.get()} is one of the files nested inside of the * {@code destination} directory. */ - static ReportFiles createReportFiles( + static FilesWithEntryPoint readFilesWithEntryPoint( File destination, Optional entryPointHint, Path rootDir) { Path destinationPath = rootDir.relativize(toNormalizedPath(destination)); if (destination.isFile()) { // The destination is a single file - find its root, and add this single file to the - // ReportFiles. - return ReportFiles.createSingleFile(destinationPath, toByteArraySupplier(destination)); + // FilesWithEntryPoint. + return FilesWithEntryPoint.createSingleFile( + destinationPath, toByteArraySupplier(destination)); } if (!destination.isDirectory()) { - // This isn't a file nor a directory - so it doesn't exist! Return empty ReportFiles - return ReportFiles.create(ImmutableMap.of(), destinationPath); + // This isn't a file nor a directory - so it doesn't exist! Return empty FilesWithEntryPoint + return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath); } // The destination is a directory - find all the actual files first @@ -170,13 +170,13 @@ final class GcsPluginUtils { file -> toByteArraySupplier(file))); if (files.isEmpty()) { - // The directory exists, but is empty. Return empty ReportFiles - return ReportFiles.create(ImmutableMap.of(), destinationPath); + // The directory exists, but is empty. Return empty FilesWithEntryPoint + return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath); } if (files.size() == 1) { // We got a directory, but it only has a single file. We can link to that. - return ReportFiles.create(files, getOnlyElement(files.keySet())); + return FilesWithEntryPoint.create(files, getOnlyElement(files.keySet())); } // There are multiple files in the report! We need to check the entryPointHint @@ -185,13 +185,13 @@ final class GcsPluginUtils { if (entryPointPath.isPresent() && files.containsKey(entryPointPath.get())) { // We were given the entry point! Use it! - return ReportFiles.create(files, entryPointPath.get()); + return FilesWithEntryPoint.create(files, entryPointPath.get()); } // We weren't given an appropriate entry point. But we still need a single link to all this data // - so we'll zip it and just host a single file. Path zipFilePath = destinationPath.resolve(destinationPath.getFileName().toString() + ".zip"); - return ReportFiles.createSingleFile(zipFilePath, createZippedByteArraySupplier(files)); + return FilesWithEntryPoint.createSingleFile(zipFilePath, createZippedByteArraySupplier(files)); } static Supplier createZippedByteArraySupplier(Map> files) { diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java index f39d8e240..68eaa8bdb 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java @@ -14,14 +14,10 @@ package google.registry.gradle.plugin; -import static com.google.common.base.Preconditions.checkArgument; - import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import google.registry.gradle.plugin.ProjectData.TaskData; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; -import java.nio.file.Path; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -107,11 +103,11 @@ abstract class ProjectData { abstract Optional> log(); /** - * Returns the ReportFiles for every report, keyed on the report type. + * Returns the FilesWithEntryPoint for every report, keyed on the report type. * *

The "html" report type is the most interesting, but there are other report formats. */ - abstract ImmutableMap reports(); + abstract ImmutableMap reports(); abstract Builder toBuilder(); @@ -129,45 +125,14 @@ abstract class ProjectData { abstract Builder setLog(Supplier log); - abstract ImmutableMap.Builder reportsBuilder(); + abstract ImmutableMap.Builder reportsBuilder(); - Builder putReport(String type, ReportFiles reportFiles) { + Builder putReport(String type, FilesWithEntryPoint reportFiles) { reportsBuilder().put(type, reportFiles); return this; } abstract TaskData build(); } - - /** The files for a single format of a specific Task. */ - @AutoValue - abstract static class ReportFiles { - - /** - * All files generated by this report, keyed from their path to a supplier of their content. - * - *

The reason we use a supplier instead of loading the content is in case the content is - * very large... - * - *

Also, no point in doing IO before we need it! - */ - abstract ImmutableMap> files(); - - /** - * The file that gives access (links...) to all the data in the report. - * - *

Guaranteed to be a key in {@link #files} if and only if files isn't empty. - */ - abstract Path entryPoint(); - - static ReportFiles create(ImmutableMap> files, Path entryPoint) { - checkArgument(files.isEmpty() || files.containsKey(entryPoint)); - return new AutoValue_ProjectData_TaskData_ReportFiles(files, entryPoint); - } - - static ReportFiles createSingleFile(Path path, Supplier data) { - return create(ImmutableMap.of(path, data), path); - } - } } } diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java similarity index 64% rename from gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java rename to gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java index 4d06eaf5a..a7dd65cf0 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java @@ -15,9 +15,9 @@ package google.registry.gradle.plugin; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; -import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles; +import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint; import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; @@ -27,6 +27,7 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; import google.registry.gradle.plugin.ProjectData.TaskData; import java.io.File; import java.io.FileInputStream; @@ -38,6 +39,7 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.HashMap; import java.util.Optional; +import java.util.function.BiConsumer; import java.util.function.Supplier; import org.gradle.api.DefaultTask; import org.gradle.api.Project; @@ -51,20 +53,32 @@ import org.gradle.api.tasks.TaskAction; /** * A task that uploads the Reports generated by other tasks to GCS. */ -public class GcsReportUploader extends DefaultTask { +public class ReportUploader extends DefaultTask { - private static final SecureRandom secureRandom = new SecureRandom(); + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + private static final ImmutableMap> UPLOAD_FUNCTIONS = + ImmutableMap.of( + "file://", ReportUploader::saveResultsToLocalFolder, + "gcs://", ReportUploader::uploadResultsToGcs); private final ArrayList tasks = new ArrayList<>(); private final HashMap logs = new HashMap<>(); private Project project; - private String bucket = null; + private String destination = null; private String credentialsFile = null; private String multithreadedUpload = null; - public void setBucket(String bucket) { - this.bucket = bucket; + /** Sets the destination of the reports. + * + * Currently supports two types of destinations: + * + * - file://[absulute local path], e.g. file:///tmp/buildOutputs/ + * + * - gcs://[bucket name]/[optional path], e.g. gcs://my-bucket/buildOutputs/ + */ + public void setDestination(String destination) { + this.destination = destination; } public void setCredentialsFile(String credentialsFile) { @@ -144,34 +158,85 @@ public class GcsReportUploader extends DefaultTask { : Optional.empty(); builder .reportsBuilder() - .put(type, createReportFiles(destination, entryPointHint, rootDir)); + .put(type, readFilesWithEntryPoint(destination, entryPointHint, rootDir)); }); } return builder.build(); } + private FilesWithEntryPoint generateFilesToUpload() { + ProjectData projectData = createProjectData(); + CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData); + return coverPageGenerator.getFilesToUpload(); + } + @TaskAction void uploadResults() { - System.out.format("GcsReportUploader: bucket= '%s'\n", bucket); - if (isNullOrEmpty(bucket)) { - System.out.format("GcsReportUploader: no bucket defined. Skipping upload\n"); - return; - } + System.out.format("ReportUploader: destination= '%s'\n", destination); try { - uploadResultsToGcs(); + + if (isNullOrEmpty(destination)) { + System.out.format("ReportUploader: no destination given, skipping...\n"); + return; + } + + for (String key : UPLOAD_FUNCTIONS.keySet()) { + if (destination.startsWith(key)) { + UPLOAD_FUNCTIONS.get(key).accept(this, destination.substring(key.length())); + return; + } + } + System.out.format( + "ReportUploader: given destination '%s' doesn't start with one of %s." + + " Defaulting to saving in /tmp\n", + destination, UPLOAD_FUNCTIONS.keySet()); + saveResultsToLocalFolder("/tmp/"); } catch (Throwable e) { - System.out.format("GcsReportUploader: Encountered error %s\n", e); + System.out.format("ReportUploader: Encountered error %s\n", e); e.printStackTrace(System.out); - System.out.format("GcsReportUploader: skipping upload\n"); + System.out.format("ReportUploader: skipping upload\n"); } } - void uploadResultsToGcs() { - checkNotNull(bucket); - ProjectData projectData = createProjectData(); + private void saveResultsToLocalFolder(String absoluteFolderName) { + Path folder = Paths.get(absoluteFolderName, createUniqueFolderName()); + checkArgument( + folder.isAbsolute(), + "Local files destination must be an absolute path, but is %s", + absoluteFolderName); + FilesWithEntryPoint filesToUpload = generateFilesToUpload(); + System.out.format( + "ReportUploader: going to save %s files to %s\n", + filesToUpload.files().size(), folder); + filesToUpload + .files() + .forEach((path, dataSupplier) -> saveFile(folder.resolve(path), dataSupplier)); + System.out.format( + "ReportUploader: report saved to file://%s\n", + folder.resolve(filesToUpload.entryPoint())); + } - Path folder = Paths.get(createUniqueFolderName()); + private void saveFile(Path path, Supplier dataSupplier) { + File dir = path.getParent().toFile(); + if (!dir.isDirectory()) { + checkState(dir.mkdirs(), "Couldn't create directory %s", dir); + } + try { + Files.write(dataSupplier.get(), path.toFile()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + + private void uploadResultsToGcs(String destination) { + checkArgument( + !destination.isEmpty(), + "destination must include at least the bucket name, but is empty"); + Path bucketWithFolder = Paths.get(destination, createUniqueFolderName()); + String bucket = bucketWithFolder.getName(0).toString(); + Path folder = bucketWithFolder.subpath(1, bucketWithFolder.getNameCount()); StorageOptions.Builder storageOptions = StorageOptions.newBuilder(); if (!isNullOrEmpty(credentialsFile)) { @@ -184,26 +249,25 @@ public class GcsReportUploader extends DefaultTask { } Storage storage = storageOptions.build().getService(); - CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData); - ImmutableMap> filesToUpload = coverPageGenerator.getFilesToUpload(); + FilesWithEntryPoint filesToUpload = generateFilesToUpload(); System.out.format( - "GcsReportUploader: going to upload %s files to %s/%s\n", - filesToUpload.size(), bucket, folder); + "ReportUploader: going to upload %s files to %s/%s\n", + filesToUpload.files().size(), bucket, folder); if ("yes".equals(multithreadedUpload)) { - System.out.format("GcsReportUploader: multi-threaded upload\n"); - uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload); + System.out.format("ReportUploader: multi-threaded upload\n"); + uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload.files()); } else { - System.out.format("GcsReportUploader: single threaded upload\n"); - filesToUpload.forEach( + System.out.format("ReportUploader: single threaded upload\n"); + filesToUpload.files().forEach( (path, dataSupplier) -> { - System.out.format("GcsReportUploader: Uploading %s\n", path); + System.out.format("ReportUploader: Uploading %s\n", path); uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier); }); } System.out.format( - "GcsReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n", - bucket, folder.resolve(coverPageGenerator.getEntryPoint())); + "ReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n", + bucket, folder.resolve(filesToUpload.entryPoint())); } void setProject(Project project) { @@ -215,7 +279,7 @@ public class GcsReportUploader extends DefaultTask { } private void addTask(Task task) { - if (task instanceof GcsReportUploader) { + if (task instanceof ReportUploader) { return; } tasks.add(task); @@ -241,9 +305,9 @@ public class GcsReportUploader extends DefaultTask { private String createUniqueFolderName() { return String.format( "%h-%h-%h-%h", - secureRandom.nextInt(), - secureRandom.nextInt(), - secureRandom.nextInt(), - secureRandom.nextInt()); + SECURE_RANDOM.nextInt(), + SECURE_RANDOM.nextInt(), + SECURE_RANDOM.nextInt(), + SECURE_RANDOM.nextInt()); } } diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java similarity index 72% rename from gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java rename to gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java index bb6dda83c..a940b3359 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java @@ -18,20 +18,20 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; /** - * Plugin setting up the GcsReportUploader task. + * Plugin setting up the ReportUploader task. * - *

It goes over all the tasks in a project and pass them on to the GcsReportUploader task for set + *

It goes over all the tasks in a project and pass them on to the ReportUploader task for set * up. * - *

Note that since we're passing in all the projects' tasks - this includes the GcsReportUploader - * itself! It's up to the GcsReportuploader to take care of not having "infinite loops" caused by + *

Note that since we're passing in all the projects' tasks - this includes the ReportUploader + * itself! It's up to the ReportUploader to take care of not having "infinite loops" caused by * waiting for itself to end before finishing. */ -public class GcsReportUploaderPlugin implements Plugin { +public class ReportUploaderPlugin implements Plugin { public void apply(Project project) { - GcsReportUploader reportUploader = - project.getTasks().create("gcsReportUploader", GcsReportUploader.class, task -> { + ReportUploader reportUploader = + project.getTasks().create("reportUploader", ReportUploader.class, task -> { task.setDescription("Uploads the reports to GCS bucket"); task.setGroup("uploads"); }); diff --git a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java index b5f077585..26c7f193d 100644 --- a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java +++ b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java @@ -23,10 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import google.registry.gradle.plugin.ProjectData.TaskData; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; -import java.nio.file.Path; import java.nio.file.Paths; -import java.util.function.Supplier; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -68,8 +65,8 @@ public final class CoverPageGeneratorTest { private ImmutableMap getGeneratedFiles(ProjectData project) { CoverPageGenerator coverPageGenerator = new CoverPageGenerator(project); - ImmutableMap> files = coverPageGenerator.getFilesToUpload(); - return files.entrySet().stream() + FilesWithEntryPoint files = coverPageGenerator.getFilesToUpload(); + return files.files().entrySet().stream() .collect( toImmutableMap( entry -> entry.getKey().toString(), @@ -87,9 +84,10 @@ public final class CoverPageGeneratorTest { } @Test - public void testGetFilesToUpload_getEntryPoint_isIndexHtml() { + public void testGetFilesToUpload_entryPoint_isIndexHtml() { CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT); - assertThat(coverPageGenerator.getEntryPoint()).isEqualTo(Paths.get("index.html")); + assertThat(coverPageGenerator.getFilesToUpload().entryPoint()) + .isEqualTo(Paths.get("index.html")); } @Test @@ -220,7 +218,7 @@ public final class CoverPageGeneratorTest { .toBuilder() .putReport( "someReport", - ReportFiles.create( + FilesWithEntryPoint.create( ImmutableMap.of( Paths.get("path", "report.txt"), toByteArraySupplier("report content")), @@ -242,7 +240,8 @@ public final class CoverPageGeneratorTest { .toBuilder() .putReport( "someReport", - ReportFiles.create(ImmutableMap.of(), Paths.get("path", "report.txt"))) + FilesWithEntryPoint.create( + ImmutableMap.of(), Paths.get("path", "report.txt"))) .build()) .build()); assertThat(files).doesNotContainKey("path/report.txt"); @@ -263,7 +262,7 @@ public final class CoverPageGeneratorTest { .setLog(toByteArraySupplier("log data")) .putReport( "filledReport", - ReportFiles.create( + FilesWithEntryPoint.create( ImmutableMap.of( Paths.get("path-filled", "report.txt"), toByteArraySupplier("report content"), @@ -272,7 +271,7 @@ public final class CoverPageGeneratorTest { Paths.get("path-filled", "report.txt"))) .putReport( "emptyReport", - ReportFiles.create( + FilesWithEntryPoint.create( ImmutableMap.of(), Paths.get("path-empty", "report.txt"))) .build()) .build()); diff --git a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java index 151a4ff6e..c087a1c18 100644 --- a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java +++ b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java @@ -16,8 +16,8 @@ package google.registry.gradle.plugin; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.truth.Truth.assertThat; -import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles; import static google.registry.gradle.plugin.GcsPluginUtils.getContentType; +import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint; import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; @@ -30,7 +30,6 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.common.collect.ImmutableMap; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; @@ -136,7 +135,7 @@ public final class GcsPluginUtilsTest { assertThat(toByteArraySupplier(file).get()).isEqualTo("some data".getBytes(UTF_8)); } - private ImmutableMap readAllFiles(ReportFiles reportFiles) { + private ImmutableMap readAllFiles(FilesWithEntryPoint reportFiles) { return reportFiles.files().entrySet().stream() .collect( toImmutableMap( @@ -153,7 +152,8 @@ public final class GcsPluginUtilsTest { // Since the entry point is obvious here - any hint given is just ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/file.txt"); assertThat(readAllFiles(files)).containsExactly("some/path/file.txt", "some data"); @@ -168,7 +168,8 @@ public final class GcsPluginUtilsTest { // Since there are not files, any hint given is obvioulsy wrong and will be ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("non/existing.txt"); assertThat(files.files()).isEmpty(); @@ -183,7 +184,8 @@ public final class GcsPluginUtilsTest { // Since there are not files, any hint given is obvioulsy wrong and will be ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path"); assertThat(files.files()).isEmpty(); @@ -200,7 +202,8 @@ public final class GcsPluginUtilsTest { // Since the entry point is obvious here - any hint given is just ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/a/file.txt"); assertThat(readAllFiles(files)).containsExactly("some/path/a/file.txt", "some data"); @@ -227,7 +230,7 @@ public final class GcsPluginUtilsTest { Files.write( folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); - ReportFiles files = createReportFiles(destination, Optional.empty(), root); + FilesWithEntryPoint files = readFilesWithEntryPoint(destination, Optional.empty(), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip"); assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip"); @@ -255,7 +258,8 @@ public final class GcsPluginUtilsTest { Files.write( folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); - ReportFiles files = createReportFiles(destination, Optional.of(badEntryPoint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(badEntryPoint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip"); assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip"); @@ -278,7 +282,8 @@ public final class GcsPluginUtilsTest { Files.write( folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); - ReportFiles files = createReportFiles(destination, Optional.of(goodEntryPoint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(goodEntryPoint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/index.html"); assertThat(readAllFiles(files)) diff --git a/gradle/gradle.properties b/gradle/gradle.properties index 6b9caf68d..281272e87 100644 --- a/gradle/gradle.properties +++ b/gradle/gradle.properties @@ -1,6 +1,6 @@ repositoryUrl= publishUrl= -gcsBucket= -gcsCredentialsFile= -gcsMultithreadedUpload= +uploaderDestination= +uploaderCredentialsFile= +uploaderMultithreadedUpload= mavenCoordinateFile=