diff --git a/.travis.yml b/.travis.yml index 12b1b420f..54a9f6cb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,9 @@ # "CREDZ" and "REPORT_GCS_BUCKET" environment variables in your travis # repository. # -# The report bucket id should just be the bucket id without any scheme or path -# (e.g. "my-bucket-name", not "gcs://my-bucket-name"). You'll want to select -# "Display value in build log", otherwise travis will hide the bucket name in -# the URL that is displayed. +# The report destination can be any GCS path (e.g. "gcs://bucket-name/path"). +# You'll want to select "Display value in build log", otherwise travis will +# hide the bucket name in the URL that is displayed. # # The CREDZ variable should be the contents of a json credentials file for # a service account with write access to the bucket, escaped for bash shell @@ -56,4 +55,4 @@ env: # output, instead of the default 10. # See notes on the CREDZ and REPORT_GCS_BUCKET environment variable in the # comments at the top of the file. -script: cd gradle && echo "$CREDZ" >credz.json && chmod 755 ./gradlew && travis_wait 45 ./gradlew build --continue -P gcsBucket="$REPORT_GCS_BUCKET" -P gcsCredentialsFile=credz.json -P gcsMultithreadedUpload=yes +script: cd gradle && echo "$CREDZ" >credz.json && chmod 755 ./gradlew && travis_wait 45 ./gradlew build --continue -P uploaderDestination="$REPORT_GCS_DESTINATION" -P uploaderCredentialsFile=credz.json -P uploaderMultithreadedUpload=yes diff --git a/gradle/build.gradle b/gradle/build.gradle index b7a8e8d36..0227127cc 100644 --- a/gradle/build.gradle +++ b/gradle/build.gradle @@ -23,14 +23,14 @@ plugins { id "com.moowork.node" version "1.2.0" } -apply plugin: google.registry.gradle.plugin.GcsReportUploaderPlugin +apply plugin: google.registry.gradle.plugin.ReportUploaderPlugin -gcsReportUploader { - // Set the bucket here to upload build results to a GCS bucket. - // e.g. -P gcsBucket=domain-registry-alpha-build-result-test +reportUploader { + // Set the location where we want to upload the build results. + // e.g. -P uploaderDestination=gcs://domain-registry-alpha-build-result-test // - // If no bucket it set - the uploading will be skipped. - bucket = gcsBucket + // If not set - the upload will be skipped + destination = uploaderDestination // The location of the file containing the OAuth2 Google Cloud credentials. // @@ -39,11 +39,11 @@ gcsReportUploader { // supported by the Cloud SDK. // // If no file is given - the default credentials are used. - credentialsFile = gcsCredentialsFile + credentialsFile = uploaderCredentialsFile // If set to 'yes', each file will be uploaded to GCS in a separate thread. // This is MUCH faster. - multithreadedUpload = gcsMultithreadedUpload + multithreadedUpload = uploaderMultithreadedUpload } apply from: 'dependencies.gradle' diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java index f71c60607..b29cf794c 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/CoverPageGenerator.java @@ -46,6 +46,8 @@ final class CoverPageGenerator { /** List of all resource files that will be uploaded as-is. */ private static final ImmutableSet STATIC_RESOURCE_FILES = ImmutableSet.of(Paths.get("css", "style.css")); + /** Name of the entry-point file that will be created. */ + private static final Path ENTRY_POINT = Paths.get("index.html"); private final ProjectData projectData; private final ImmutableSetMultimap tasksByState; @@ -63,30 +65,19 @@ final class CoverPageGenerator { projectData.tasks().stream().collect(toImmutableSetMultimap(TaskData::state, task -> task)); } - /** - * The (relative) entry point for the cover page. - * - *

A file with this relative path is guaranteed to be returned from {@link #getFilesToUpload}, - * and a browser pointing to this page will have access to all the data generated by {@link - * #getFilesToUpload}. - */ - Path getEntryPoint() { - return Paths.get("index.html"); - } - /** * Returns all the files that need uploading for the cover page to work. * *

This includes all the report files as well, to make sure that the link works. */ - ImmutableMap> getFilesToUpload() { + FilesWithEntryPoint getFilesToUpload() { ImmutableMap.Builder> builder = new ImmutableMap.Builder<>(); // Add all the static resource pages STATIC_RESOURCE_FILES.stream().forEach(file -> builder.put(file, resourceLoader(file))); // Create the cover page // Note that the ByteArraySupplier here is lazy - the createCoverPage function is only called // when the resulting Supplier's get function is called. - builder.put(getEntryPoint(), toByteArraySupplier(this::createCoverPage)); + builder.put(ENTRY_POINT, toByteArraySupplier(this::createCoverPage)); // Add all the files from the tasks tasksByState.values().stream() .flatMap(task -> task.reports().values().stream()) @@ -96,7 +87,7 @@ final class CoverPageGenerator { .filter(task -> task.log().isPresent()) .forEach(task -> builder.put(getLogPath(task), task.log().get())); - return builder.build(); + return FilesWithEntryPoint.create(builder.build(), ENTRY_POINT); } /** Renders the cover page. */ diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java new file mode 100644 index 000000000..f8f098b38 --- /dev/null +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/FilesWithEntryPoint.java @@ -0,0 +1,59 @@ +// Copyright 2019 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.gradle.plugin; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import java.nio.file.Path; +import java.util.function.Supplier; + +/** + * Holds a set of files with a browser-friendly entry point to those files. + * + *

The file data is lazily generated. + * + *

If there is at least one file, it's guaranteed that the entry point is one of these files. + */ +@AutoValue +abstract class FilesWithEntryPoint { + + /** + * All files that are part of this report, keyed from their path to a supplier of their content. + * + *

The reason we use a supplier instead of loading the content is in case the content is very + * large... + * + *

Also, no point in doing IO before we need it! + */ + abstract ImmutableMap> files(); + + /** + * The file that gives access (links...) to all the data in the report. + * + *

Guaranteed to be a key in {@link #files} if and only if files isn't empty. + */ + abstract Path entryPoint(); + + static FilesWithEntryPoint create(ImmutableMap> files, Path entryPoint) { + checkArgument(files.isEmpty() || files.containsKey(entryPoint)); + return new AutoValue_FilesWithEntryPoint(files, entryPoint); + } + + static FilesWithEntryPoint createSingleFile(Path path, Supplier data) { + return create(ImmutableMap.of(path, data), path); + } +} diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java index 99fe3c555..afb47b9a5 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsPluginUtils.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; import com.google.common.io.Files; import com.google.common.io.Resources; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -123,10 +122,10 @@ final class GcsPluginUtils { } /** - * Reads all the files generated by a Report into a ReportFiles object. + * Reads all the files generated by a Report into a FilesWithEntryPoint object. * - *

Every ReportFiles must have a single link "entry point" that gives users access to all the - * files. If the report generated just one file - we will just link to that file. + *

Every FilesWithEntryPoint must have a single link "entry point" that gives users access to + * all the files. If the report generated just one file - we will just link to that file. * *

However, if the report generated more than one file - the only thing we can safely do is to * zip all the files and link to the zip file. @@ -144,20 +143,21 @@ final class GcsPluginUtils { * directory, and (c) {@code entryPointHint.get()} is one of the files nested inside of the * {@code destination} directory. */ - static ReportFiles createReportFiles( + static FilesWithEntryPoint readFilesWithEntryPoint( File destination, Optional entryPointHint, Path rootDir) { Path destinationPath = rootDir.relativize(toNormalizedPath(destination)); if (destination.isFile()) { // The destination is a single file - find its root, and add this single file to the - // ReportFiles. - return ReportFiles.createSingleFile(destinationPath, toByteArraySupplier(destination)); + // FilesWithEntryPoint. + return FilesWithEntryPoint.createSingleFile( + destinationPath, toByteArraySupplier(destination)); } if (!destination.isDirectory()) { - // This isn't a file nor a directory - so it doesn't exist! Return empty ReportFiles - return ReportFiles.create(ImmutableMap.of(), destinationPath); + // This isn't a file nor a directory - so it doesn't exist! Return empty FilesWithEntryPoint + return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath); } // The destination is a directory - find all the actual files first @@ -170,13 +170,13 @@ final class GcsPluginUtils { file -> toByteArraySupplier(file))); if (files.isEmpty()) { - // The directory exists, but is empty. Return empty ReportFiles - return ReportFiles.create(ImmutableMap.of(), destinationPath); + // The directory exists, but is empty. Return empty FilesWithEntryPoint + return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath); } if (files.size() == 1) { // We got a directory, but it only has a single file. We can link to that. - return ReportFiles.create(files, getOnlyElement(files.keySet())); + return FilesWithEntryPoint.create(files, getOnlyElement(files.keySet())); } // There are multiple files in the report! We need to check the entryPointHint @@ -185,13 +185,13 @@ final class GcsPluginUtils { if (entryPointPath.isPresent() && files.containsKey(entryPointPath.get())) { // We were given the entry point! Use it! - return ReportFiles.create(files, entryPointPath.get()); + return FilesWithEntryPoint.create(files, entryPointPath.get()); } // We weren't given an appropriate entry point. But we still need a single link to all this data // - so we'll zip it and just host a single file. Path zipFilePath = destinationPath.resolve(destinationPath.getFileName().toString() + ".zip"); - return ReportFiles.createSingleFile(zipFilePath, createZippedByteArraySupplier(files)); + return FilesWithEntryPoint.createSingleFile(zipFilePath, createZippedByteArraySupplier(files)); } static Supplier createZippedByteArraySupplier(Map> files) { diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java index f39d8e240..68eaa8bdb 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java @@ -14,14 +14,10 @@ package google.registry.gradle.plugin; -import static com.google.common.base.Preconditions.checkArgument; - import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import google.registry.gradle.plugin.ProjectData.TaskData; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; -import java.nio.file.Path; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -107,11 +103,11 @@ abstract class ProjectData { abstract Optional> log(); /** - * Returns the ReportFiles for every report, keyed on the report type. + * Returns the FilesWithEntryPoint for every report, keyed on the report type. * *

The "html" report type is the most interesting, but there are other report formats. */ - abstract ImmutableMap reports(); + abstract ImmutableMap reports(); abstract Builder toBuilder(); @@ -129,45 +125,14 @@ abstract class ProjectData { abstract Builder setLog(Supplier log); - abstract ImmutableMap.Builder reportsBuilder(); + abstract ImmutableMap.Builder reportsBuilder(); - Builder putReport(String type, ReportFiles reportFiles) { + Builder putReport(String type, FilesWithEntryPoint reportFiles) { reportsBuilder().put(type, reportFiles); return this; } abstract TaskData build(); } - - /** The files for a single format of a specific Task. */ - @AutoValue - abstract static class ReportFiles { - - /** - * All files generated by this report, keyed from their path to a supplier of their content. - * - *

The reason we use a supplier instead of loading the content is in case the content is - * very large... - * - *

Also, no point in doing IO before we need it! - */ - abstract ImmutableMap> files(); - - /** - * The file that gives access (links...) to all the data in the report. - * - *

Guaranteed to be a key in {@link #files} if and only if files isn't empty. - */ - abstract Path entryPoint(); - - static ReportFiles create(ImmutableMap> files, Path entryPoint) { - checkArgument(files.isEmpty() || files.containsKey(entryPoint)); - return new AutoValue_ProjectData_TaskData_ReportFiles(files, entryPoint); - } - - static ReportFiles createSingleFile(Path path, Supplier data) { - return create(ImmutableMap.of(path, data), path); - } - } } } diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java similarity index 64% rename from gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java rename to gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java index 4d06eaf5a..a7dd65cf0 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploader.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploader.java @@ -15,9 +15,9 @@ package google.registry.gradle.plugin; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Strings.isNullOrEmpty; -import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles; +import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint; import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; @@ -27,6 +27,7 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; import google.registry.gradle.plugin.ProjectData.TaskData; import java.io.File; import java.io.FileInputStream; @@ -38,6 +39,7 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.HashMap; import java.util.Optional; +import java.util.function.BiConsumer; import java.util.function.Supplier; import org.gradle.api.DefaultTask; import org.gradle.api.Project; @@ -51,20 +53,32 @@ import org.gradle.api.tasks.TaskAction; /** * A task that uploads the Reports generated by other tasks to GCS. */ -public class GcsReportUploader extends DefaultTask { +public class ReportUploader extends DefaultTask { - private static final SecureRandom secureRandom = new SecureRandom(); + private static final SecureRandom SECURE_RANDOM = new SecureRandom(); + private static final ImmutableMap> UPLOAD_FUNCTIONS = + ImmutableMap.of( + "file://", ReportUploader::saveResultsToLocalFolder, + "gcs://", ReportUploader::uploadResultsToGcs); private final ArrayList tasks = new ArrayList<>(); private final HashMap logs = new HashMap<>(); private Project project; - private String bucket = null; + private String destination = null; private String credentialsFile = null; private String multithreadedUpload = null; - public void setBucket(String bucket) { - this.bucket = bucket; + /** Sets the destination of the reports. + * + * Currently supports two types of destinations: + * + * - file://[absulute local path], e.g. file:///tmp/buildOutputs/ + * + * - gcs://[bucket name]/[optional path], e.g. gcs://my-bucket/buildOutputs/ + */ + public void setDestination(String destination) { + this.destination = destination; } public void setCredentialsFile(String credentialsFile) { @@ -144,34 +158,85 @@ public class GcsReportUploader extends DefaultTask { : Optional.empty(); builder .reportsBuilder() - .put(type, createReportFiles(destination, entryPointHint, rootDir)); + .put(type, readFilesWithEntryPoint(destination, entryPointHint, rootDir)); }); } return builder.build(); } + private FilesWithEntryPoint generateFilesToUpload() { + ProjectData projectData = createProjectData(); + CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData); + return coverPageGenerator.getFilesToUpload(); + } + @TaskAction void uploadResults() { - System.out.format("GcsReportUploader: bucket= '%s'\n", bucket); - if (isNullOrEmpty(bucket)) { - System.out.format("GcsReportUploader: no bucket defined. Skipping upload\n"); - return; - } + System.out.format("ReportUploader: destination= '%s'\n", destination); try { - uploadResultsToGcs(); + + if (isNullOrEmpty(destination)) { + System.out.format("ReportUploader: no destination given, skipping...\n"); + return; + } + + for (String key : UPLOAD_FUNCTIONS.keySet()) { + if (destination.startsWith(key)) { + UPLOAD_FUNCTIONS.get(key).accept(this, destination.substring(key.length())); + return; + } + } + System.out.format( + "ReportUploader: given destination '%s' doesn't start with one of %s." + + " Defaulting to saving in /tmp\n", + destination, UPLOAD_FUNCTIONS.keySet()); + saveResultsToLocalFolder("/tmp/"); } catch (Throwable e) { - System.out.format("GcsReportUploader: Encountered error %s\n", e); + System.out.format("ReportUploader: Encountered error %s\n", e); e.printStackTrace(System.out); - System.out.format("GcsReportUploader: skipping upload\n"); + System.out.format("ReportUploader: skipping upload\n"); } } - void uploadResultsToGcs() { - checkNotNull(bucket); - ProjectData projectData = createProjectData(); + private void saveResultsToLocalFolder(String absoluteFolderName) { + Path folder = Paths.get(absoluteFolderName, createUniqueFolderName()); + checkArgument( + folder.isAbsolute(), + "Local files destination must be an absolute path, but is %s", + absoluteFolderName); + FilesWithEntryPoint filesToUpload = generateFilesToUpload(); + System.out.format( + "ReportUploader: going to save %s files to %s\n", + filesToUpload.files().size(), folder); + filesToUpload + .files() + .forEach((path, dataSupplier) -> saveFile(folder.resolve(path), dataSupplier)); + System.out.format( + "ReportUploader: report saved to file://%s\n", + folder.resolve(filesToUpload.entryPoint())); + } - Path folder = Paths.get(createUniqueFolderName()); + private void saveFile(Path path, Supplier dataSupplier) { + File dir = path.getParent().toFile(); + if (!dir.isDirectory()) { + checkState(dir.mkdirs(), "Couldn't create directory %s", dir); + } + try { + Files.write(dataSupplier.get(), path.toFile()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + + private void uploadResultsToGcs(String destination) { + checkArgument( + !destination.isEmpty(), + "destination must include at least the bucket name, but is empty"); + Path bucketWithFolder = Paths.get(destination, createUniqueFolderName()); + String bucket = bucketWithFolder.getName(0).toString(); + Path folder = bucketWithFolder.subpath(1, bucketWithFolder.getNameCount()); StorageOptions.Builder storageOptions = StorageOptions.newBuilder(); if (!isNullOrEmpty(credentialsFile)) { @@ -184,26 +249,25 @@ public class GcsReportUploader extends DefaultTask { } Storage storage = storageOptions.build().getService(); - CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData); - ImmutableMap> filesToUpload = coverPageGenerator.getFilesToUpload(); + FilesWithEntryPoint filesToUpload = generateFilesToUpload(); System.out.format( - "GcsReportUploader: going to upload %s files to %s/%s\n", - filesToUpload.size(), bucket, folder); + "ReportUploader: going to upload %s files to %s/%s\n", + filesToUpload.files().size(), bucket, folder); if ("yes".equals(multithreadedUpload)) { - System.out.format("GcsReportUploader: multi-threaded upload\n"); - uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload); + System.out.format("ReportUploader: multi-threaded upload\n"); + uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload.files()); } else { - System.out.format("GcsReportUploader: single threaded upload\n"); - filesToUpload.forEach( + System.out.format("ReportUploader: single threaded upload\n"); + filesToUpload.files().forEach( (path, dataSupplier) -> { - System.out.format("GcsReportUploader: Uploading %s\n", path); + System.out.format("ReportUploader: Uploading %s\n", path); uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier); }); } System.out.format( - "GcsReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n", - bucket, folder.resolve(coverPageGenerator.getEntryPoint())); + "ReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n", + bucket, folder.resolve(filesToUpload.entryPoint())); } void setProject(Project project) { @@ -215,7 +279,7 @@ public class GcsReportUploader extends DefaultTask { } private void addTask(Task task) { - if (task instanceof GcsReportUploader) { + if (task instanceof ReportUploader) { return; } tasks.add(task); @@ -241,9 +305,9 @@ public class GcsReportUploader extends DefaultTask { private String createUniqueFolderName() { return String.format( "%h-%h-%h-%h", - secureRandom.nextInt(), - secureRandom.nextInt(), - secureRandom.nextInt(), - secureRandom.nextInt()); + SECURE_RANDOM.nextInt(), + SECURE_RANDOM.nextInt(), + SECURE_RANDOM.nextInt(), + SECURE_RANDOM.nextInt()); } } diff --git a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java similarity index 72% rename from gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java rename to gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java index bb6dda83c..a940b3359 100644 --- a/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/GcsReportUploaderPlugin.java +++ b/gradle/buildSrc/src/main/java/google/registry/gradle/plugin/ReportUploaderPlugin.java @@ -18,20 +18,20 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; /** - * Plugin setting up the GcsReportUploader task. + * Plugin setting up the ReportUploader task. * - *

It goes over all the tasks in a project and pass them on to the GcsReportUploader task for set + *

It goes over all the tasks in a project and pass them on to the ReportUploader task for set * up. * - *

Note that since we're passing in all the projects' tasks - this includes the GcsReportUploader - * itself! It's up to the GcsReportuploader to take care of not having "infinite loops" caused by + *

Note that since we're passing in all the projects' tasks - this includes the ReportUploader + * itself! It's up to the ReportUploader to take care of not having "infinite loops" caused by * waiting for itself to end before finishing. */ -public class GcsReportUploaderPlugin implements Plugin { +public class ReportUploaderPlugin implements Plugin { public void apply(Project project) { - GcsReportUploader reportUploader = - project.getTasks().create("gcsReportUploader", GcsReportUploader.class, task -> { + ReportUploader reportUploader = + project.getTasks().create("reportUploader", ReportUploader.class, task -> { task.setDescription("Uploads the reports to GCS bucket"); task.setGroup("uploads"); }); diff --git a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java index b5f077585..26c7f193d 100644 --- a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java +++ b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/CoverPageGeneratorTest.java @@ -23,10 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import google.registry.gradle.plugin.ProjectData.TaskData; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; -import java.nio.file.Path; import java.nio.file.Paths; -import java.util.function.Supplier; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -68,8 +65,8 @@ public final class CoverPageGeneratorTest { private ImmutableMap getGeneratedFiles(ProjectData project) { CoverPageGenerator coverPageGenerator = new CoverPageGenerator(project); - ImmutableMap> files = coverPageGenerator.getFilesToUpload(); - return files.entrySet().stream() + FilesWithEntryPoint files = coverPageGenerator.getFilesToUpload(); + return files.files().entrySet().stream() .collect( toImmutableMap( entry -> entry.getKey().toString(), @@ -87,9 +84,10 @@ public final class CoverPageGeneratorTest { } @Test - public void testGetFilesToUpload_getEntryPoint_isIndexHtml() { + public void testGetFilesToUpload_entryPoint_isIndexHtml() { CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT); - assertThat(coverPageGenerator.getEntryPoint()).isEqualTo(Paths.get("index.html")); + assertThat(coverPageGenerator.getFilesToUpload().entryPoint()) + .isEqualTo(Paths.get("index.html")); } @Test @@ -220,7 +218,7 @@ public final class CoverPageGeneratorTest { .toBuilder() .putReport( "someReport", - ReportFiles.create( + FilesWithEntryPoint.create( ImmutableMap.of( Paths.get("path", "report.txt"), toByteArraySupplier("report content")), @@ -242,7 +240,8 @@ public final class CoverPageGeneratorTest { .toBuilder() .putReport( "someReport", - ReportFiles.create(ImmutableMap.of(), Paths.get("path", "report.txt"))) + FilesWithEntryPoint.create( + ImmutableMap.of(), Paths.get("path", "report.txt"))) .build()) .build()); assertThat(files).doesNotContainKey("path/report.txt"); @@ -263,7 +262,7 @@ public final class CoverPageGeneratorTest { .setLog(toByteArraySupplier("log data")) .putReport( "filledReport", - ReportFiles.create( + FilesWithEntryPoint.create( ImmutableMap.of( Paths.get("path-filled", "report.txt"), toByteArraySupplier("report content"), @@ -272,7 +271,7 @@ public final class CoverPageGeneratorTest { Paths.get("path-filled", "report.txt"))) .putReport( "emptyReport", - ReportFiles.create( + FilesWithEntryPoint.create( ImmutableMap.of(), Paths.get("path-empty", "report.txt"))) .build()) .build()); diff --git a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java index 151a4ff6e..c087a1c18 100644 --- a/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java +++ b/gradle/buildSrc/src/test/java/google/registry/gradle/plugin/GcsPluginUtilsTest.java @@ -16,8 +16,8 @@ package google.registry.gradle.plugin; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.truth.Truth.assertThat; -import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles; import static google.registry.gradle.plugin.GcsPluginUtils.getContentType; +import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint; import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier; import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath; import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs; @@ -30,7 +30,6 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.common.collect.ImmutableMap; -import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; @@ -136,7 +135,7 @@ public final class GcsPluginUtilsTest { assertThat(toByteArraySupplier(file).get()).isEqualTo("some data".getBytes(UTF_8)); } - private ImmutableMap readAllFiles(ReportFiles reportFiles) { + private ImmutableMap readAllFiles(FilesWithEntryPoint reportFiles) { return reportFiles.files().entrySet().stream() .collect( toImmutableMap( @@ -153,7 +152,8 @@ public final class GcsPluginUtilsTest { // Since the entry point is obvious here - any hint given is just ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/file.txt"); assertThat(readAllFiles(files)).containsExactly("some/path/file.txt", "some data"); @@ -168,7 +168,8 @@ public final class GcsPluginUtilsTest { // Since there are not files, any hint given is obvioulsy wrong and will be ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("non/existing.txt"); assertThat(files.files()).isEmpty(); @@ -183,7 +184,8 @@ public final class GcsPluginUtilsTest { // Since there are not files, any hint given is obvioulsy wrong and will be ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path"); assertThat(files.files()).isEmpty(); @@ -200,7 +202,8 @@ public final class GcsPluginUtilsTest { // Since the entry point is obvious here - any hint given is just ignored. File ignoredHint = folder.newFile("my/root/ignored.txt"); - ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/a/file.txt"); assertThat(readAllFiles(files)).containsExactly("some/path/a/file.txt", "some data"); @@ -227,7 +230,7 @@ public final class GcsPluginUtilsTest { Files.write( folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); - ReportFiles files = createReportFiles(destination, Optional.empty(), root); + FilesWithEntryPoint files = readFilesWithEntryPoint(destination, Optional.empty(), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip"); assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip"); @@ -255,7 +258,8 @@ public final class GcsPluginUtilsTest { Files.write( folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); - ReportFiles files = createReportFiles(destination, Optional.of(badEntryPoint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(badEntryPoint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip"); assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip"); @@ -278,7 +282,8 @@ public final class GcsPluginUtilsTest { Files.write( folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8)); - ReportFiles files = createReportFiles(destination, Optional.of(goodEntryPoint), root); + FilesWithEntryPoint files = + readFilesWithEntryPoint(destination, Optional.of(goodEntryPoint), root); assertThat(files.entryPoint().toString()).isEqualTo("some/path/index.html"); assertThat(readAllFiles(files)) diff --git a/gradle/gradle.properties b/gradle/gradle.properties index 6b9caf68d..281272e87 100644 --- a/gradle/gradle.properties +++ b/gradle/gradle.properties @@ -1,6 +1,6 @@ repositoryUrl= publishUrl= -gcsBucket= -gcsCredentialsFile= -gcsMultithreadedUpload= +uploaderDestination= +uploaderCredentialsFile= +uploaderMultithreadedUpload= mavenCoordinateFile=