mirror of
https://github.com/google/nomulus.git
synced 2025-07-22 02:36:03 +02:00
Add ability to save report to local folder
Updated the plugin to receive the "protocol"-like tag in the destination, so that you can choose whether to upload to GCS or just save it locally. Possibly we might expand this in the future, but for now the goal was to allow saving our "internal" builds locally until we find a secure way to store AND BROWSE them remotely. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=238055136
This commit is contained in:
guyben
Ben McIlwain
parent
bc3bdc7347
commit
a03d10ce22
11 changed files with 229 additions and 147 deletions
|
|
@ -16,10 +16,9 @@
|
|||
# "CREDZ" and "REPORT_GCS_BUCKET" environment variables in your travis
|
||||
# repository.
|
||||
#
|
||||
# The report bucket id should just be the bucket id without any scheme or path
|
||||
# (e.g. "my-bucket-name", not "gcs://my-bucket-name"). You'll want to select
|
||||
# "Display value in build log", otherwise travis will hide the bucket name in
|
||||
# the URL that is displayed.
|
||||
# The report destination can be any GCS path (e.g. "gcs://bucket-name/path").
|
||||
# You'll want to select "Display value in build log", otherwise travis will
|
||||
# hide the bucket name in the URL that is displayed.
|
||||
#
|
||||
# The CREDZ variable should be the contents of a json credentials file for
|
||||
# a service account with write access to the bucket, escaped for bash shell
|
||||
|
|
@ -56,4 +55,4 @@ env:
|
|||
# output, instead of the default 10.
|
||||
# See notes on the CREDZ and REPORT_GCS_BUCKET environment variable in the
|
||||
# comments at the top of the file.
|
||||
script: cd gradle && echo "$CREDZ" >credz.json && chmod 755 ./gradlew && travis_wait 45 ./gradlew build --continue -P gcsBucket="$REPORT_GCS_BUCKET" -P gcsCredentialsFile=credz.json -P gcsMultithreadedUpload=yes
|
||||
script: cd gradle && echo "$CREDZ" >credz.json && chmod 755 ./gradlew && travis_wait 45 ./gradlew build --continue -P uploaderDestination="$REPORT_GCS_DESTINATION" -P uploaderCredentialsFile=credz.json -P uploaderMultithreadedUpload=yes
|
||||
|
|
|
|||
|
|
@ -23,14 +23,14 @@ plugins {
|
|||
id "com.moowork.node" version "1.2.0"
|
||||
}
|
||||
|
||||
apply plugin: google.registry.gradle.plugin.GcsReportUploaderPlugin
|
||||
apply plugin: google.registry.gradle.plugin.ReportUploaderPlugin
|
||||
|
||||
gcsReportUploader {
|
||||
// Set the bucket here to upload build results to a GCS bucket.
|
||||
// e.g. -P gcsBucket=domain-registry-alpha-build-result-test
|
||||
reportUploader {
|
||||
// Set the location where we want to upload the build results.
|
||||
// e.g. -P uploaderDestination=gcs://domain-registry-alpha-build-result-test
|
||||
//
|
||||
// If no bucket it set - the uploading will be skipped.
|
||||
bucket = gcsBucket
|
||||
// If not set - the upload will be skipped
|
||||
destination = uploaderDestination
|
||||
|
||||
// The location of the file containing the OAuth2 Google Cloud credentials.
|
||||
//
|
||||
|
|
@ -39,11 +39,11 @@ gcsReportUploader {
|
|||
// supported by the Cloud SDK.
|
||||
//
|
||||
// If no file is given - the default credentials are used.
|
||||
credentialsFile = gcsCredentialsFile
|
||||
credentialsFile = uploaderCredentialsFile
|
||||
|
||||
// If set to 'yes', each file will be uploaded to GCS in a separate thread.
|
||||
// This is MUCH faster.
|
||||
multithreadedUpload = gcsMultithreadedUpload
|
||||
multithreadedUpload = uploaderMultithreadedUpload
|
||||
}
|
||||
|
||||
apply from: 'dependencies.gradle'
|
||||
|
|
|
|||
|
|
@ -46,6 +46,8 @@ final class CoverPageGenerator {
|
|||
/** List of all resource files that will be uploaded as-is. */
|
||||
private static final ImmutableSet<Path> STATIC_RESOURCE_FILES =
|
||||
ImmutableSet.of(Paths.get("css", "style.css"));
|
||||
/** Name of the entry-point file that will be created. */
|
||||
private static final Path ENTRY_POINT = Paths.get("index.html");
|
||||
|
||||
private final ProjectData projectData;
|
||||
private final ImmutableSetMultimap<TaskData.State, TaskData> tasksByState;
|
||||
|
|
@ -63,30 +65,19 @@ final class CoverPageGenerator {
|
|||
projectData.tasks().stream().collect(toImmutableSetMultimap(TaskData::state, task -> task));
|
||||
}
|
||||
|
||||
/**
|
||||
* The (relative) entry point for the cover page.
|
||||
*
|
||||
* <p>A file with this relative path is guaranteed to be returned from {@link #getFilesToUpload},
|
||||
* and a browser pointing to this page will have access to all the data generated by {@link
|
||||
* #getFilesToUpload}.
|
||||
*/
|
||||
Path getEntryPoint() {
|
||||
return Paths.get("index.html");
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all the files that need uploading for the cover page to work.
|
||||
*
|
||||
* <p>This includes all the report files as well, to make sure that the link works.
|
||||
*/
|
||||
ImmutableMap<Path, Supplier<byte[]>> getFilesToUpload() {
|
||||
FilesWithEntryPoint getFilesToUpload() {
|
||||
ImmutableMap.Builder<Path, Supplier<byte[]>> builder = new ImmutableMap.Builder<>();
|
||||
// Add all the static resource pages
|
||||
STATIC_RESOURCE_FILES.stream().forEach(file -> builder.put(file, resourceLoader(file)));
|
||||
// Create the cover page
|
||||
// Note that the ByteArraySupplier here is lazy - the createCoverPage function is only called
|
||||
// when the resulting Supplier's get function is called.
|
||||
builder.put(getEntryPoint(), toByteArraySupplier(this::createCoverPage));
|
||||
builder.put(ENTRY_POINT, toByteArraySupplier(this::createCoverPage));
|
||||
// Add all the files from the tasks
|
||||
tasksByState.values().stream()
|
||||
.flatMap(task -> task.reports().values().stream())
|
||||
|
|
@ -96,7 +87,7 @@ final class CoverPageGenerator {
|
|||
.filter(task -> task.log().isPresent())
|
||||
.forEach(task -> builder.put(getLogPath(task), task.log().get()));
|
||||
|
||||
return builder.build();
|
||||
return FilesWithEntryPoint.create(builder.build(), ENTRY_POINT);
|
||||
}
|
||||
|
||||
/** Renders the cover page. */
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.gradle.plugin;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import java.nio.file.Path;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Holds a set of files with a browser-friendly entry point to those files.
|
||||
*
|
||||
* <p>The file data is lazily generated.
|
||||
*
|
||||
* <p>If there is at least one file, it's guaranteed that the entry point is one of these files.
|
||||
*/
|
||||
@AutoValue
|
||||
abstract class FilesWithEntryPoint {
|
||||
|
||||
/**
|
||||
* All files that are part of this report, keyed from their path to a supplier of their content.
|
||||
*
|
||||
* <p>The reason we use a supplier instead of loading the content is in case the content is very
|
||||
* large...
|
||||
*
|
||||
* <p>Also, no point in doing IO before we need it!
|
||||
*/
|
||||
abstract ImmutableMap<Path, Supplier<byte[]>> files();
|
||||
|
||||
/**
|
||||
* The file that gives access (links...) to all the data in the report.
|
||||
*
|
||||
* <p>Guaranteed to be a key in {@link #files} if and only if files isn't empty.
|
||||
*/
|
||||
abstract Path entryPoint();
|
||||
|
||||
static FilesWithEntryPoint create(ImmutableMap<Path, Supplier<byte[]>> files, Path entryPoint) {
|
||||
checkArgument(files.isEmpty() || files.containsKey(entryPoint));
|
||||
return new AutoValue_FilesWithEntryPoint(files, entryPoint);
|
||||
}
|
||||
|
||||
static FilesWithEntryPoint createSingleFile(Path path, Supplier<byte[]> data) {
|
||||
return create(ImmutableMap.of(path, data), path);
|
||||
}
|
||||
}
|
||||
|
|
@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap;
|
|||
import com.google.common.collect.Streams;
|
||||
import com.google.common.io.Files;
|
||||
import com.google.common.io.Resources;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
|
@ -123,10 +122,10 @@ final class GcsPluginUtils {
|
|||
}
|
||||
|
||||
/**
|
||||
* Reads all the files generated by a Report into a ReportFiles object.
|
||||
* Reads all the files generated by a Report into a FilesWithEntryPoint object.
|
||||
*
|
||||
* <p>Every ReportFiles must have a single link "entry point" that gives users access to all the
|
||||
* files. If the report generated just one file - we will just link to that file.
|
||||
* <p>Every FilesWithEntryPoint must have a single link "entry point" that gives users access to
|
||||
* all the files. If the report generated just one file - we will just link to that file.
|
||||
*
|
||||
* <p>However, if the report generated more than one file - the only thing we can safely do is to
|
||||
* zip all the files and link to the zip file.
|
||||
|
|
@ -144,20 +143,21 @@ final class GcsPluginUtils {
|
|||
* directory, and (c) {@code entryPointHint.get()} is one of the files nested inside of the
|
||||
* {@code destination} directory.
|
||||
*/
|
||||
static ReportFiles createReportFiles(
|
||||
static FilesWithEntryPoint readFilesWithEntryPoint(
|
||||
File destination, Optional<File> entryPointHint, Path rootDir) {
|
||||
|
||||
Path destinationPath = rootDir.relativize(toNormalizedPath(destination));
|
||||
|
||||
if (destination.isFile()) {
|
||||
// The destination is a single file - find its root, and add this single file to the
|
||||
// ReportFiles.
|
||||
return ReportFiles.createSingleFile(destinationPath, toByteArraySupplier(destination));
|
||||
// FilesWithEntryPoint.
|
||||
return FilesWithEntryPoint.createSingleFile(
|
||||
destinationPath, toByteArraySupplier(destination));
|
||||
}
|
||||
|
||||
if (!destination.isDirectory()) {
|
||||
// This isn't a file nor a directory - so it doesn't exist! Return empty ReportFiles
|
||||
return ReportFiles.create(ImmutableMap.of(), destinationPath);
|
||||
// This isn't a file nor a directory - so it doesn't exist! Return empty FilesWithEntryPoint
|
||||
return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath);
|
||||
}
|
||||
|
||||
// The destination is a directory - find all the actual files first
|
||||
|
|
@ -170,13 +170,13 @@ final class GcsPluginUtils {
|
|||
file -> toByteArraySupplier(file)));
|
||||
|
||||
if (files.isEmpty()) {
|
||||
// The directory exists, but is empty. Return empty ReportFiles
|
||||
return ReportFiles.create(ImmutableMap.of(), destinationPath);
|
||||
// The directory exists, but is empty. Return empty FilesWithEntryPoint
|
||||
return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath);
|
||||
}
|
||||
|
||||
if (files.size() == 1) {
|
||||
// We got a directory, but it only has a single file. We can link to that.
|
||||
return ReportFiles.create(files, getOnlyElement(files.keySet()));
|
||||
return FilesWithEntryPoint.create(files, getOnlyElement(files.keySet()));
|
||||
}
|
||||
|
||||
// There are multiple files in the report! We need to check the entryPointHint
|
||||
|
|
@ -185,13 +185,13 @@ final class GcsPluginUtils {
|
|||
|
||||
if (entryPointPath.isPresent() && files.containsKey(entryPointPath.get())) {
|
||||
// We were given the entry point! Use it!
|
||||
return ReportFiles.create(files, entryPointPath.get());
|
||||
return FilesWithEntryPoint.create(files, entryPointPath.get());
|
||||
}
|
||||
|
||||
// We weren't given an appropriate entry point. But we still need a single link to all this data
|
||||
// - so we'll zip it and just host a single file.
|
||||
Path zipFilePath = destinationPath.resolve(destinationPath.getFileName().toString() + ".zip");
|
||||
return ReportFiles.createSingleFile(zipFilePath, createZippedByteArraySupplier(files));
|
||||
return FilesWithEntryPoint.createSingleFile(zipFilePath, createZippedByteArraySupplier(files));
|
||||
}
|
||||
|
||||
static Supplier<byte[]> createZippedByteArraySupplier(Map<Path, Supplier<byte[]>> files) {
|
||||
|
|
|
|||
|
|
@ -14,14 +14,10 @@
|
|||
|
||||
package google.registry.gradle.plugin;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
||||
import com.google.auto.value.AutoValue;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
|
|
@ -107,11 +103,11 @@ abstract class ProjectData {
|
|||
abstract Optional<Supplier<byte[]>> log();
|
||||
|
||||
/**
|
||||
* Returns the ReportFiles for every report, keyed on the report type.
|
||||
* Returns the FilesWithEntryPoint for every report, keyed on the report type.
|
||||
*
|
||||
* <p>The "html" report type is the most interesting, but there are other report formats.
|
||||
*/
|
||||
abstract ImmutableMap<String, ReportFiles> reports();
|
||||
abstract ImmutableMap<String, FilesWithEntryPoint> reports();
|
||||
|
||||
abstract Builder toBuilder();
|
||||
|
||||
|
|
@ -129,45 +125,14 @@ abstract class ProjectData {
|
|||
|
||||
abstract Builder setLog(Supplier<byte[]> log);
|
||||
|
||||
abstract ImmutableMap.Builder<String, ReportFiles> reportsBuilder();
|
||||
abstract ImmutableMap.Builder<String, FilesWithEntryPoint> reportsBuilder();
|
||||
|
||||
Builder putReport(String type, ReportFiles reportFiles) {
|
||||
Builder putReport(String type, FilesWithEntryPoint reportFiles) {
|
||||
reportsBuilder().put(type, reportFiles);
|
||||
return this;
|
||||
}
|
||||
|
||||
abstract TaskData build();
|
||||
}
|
||||
|
||||
/** The files for a single format of a specific Task. */
|
||||
@AutoValue
|
||||
abstract static class ReportFiles {
|
||||
|
||||
/**
|
||||
* All files generated by this report, keyed from their path to a supplier of their content.
|
||||
*
|
||||
* <p>The reason we use a supplier instead of loading the content is in case the content is
|
||||
* very large...
|
||||
*
|
||||
* <p>Also, no point in doing IO before we need it!
|
||||
*/
|
||||
abstract ImmutableMap<Path, Supplier<byte[]>> files();
|
||||
|
||||
/**
|
||||
* The file that gives access (links...) to all the data in the report.
|
||||
*
|
||||
* <p>Guaranteed to be a key in {@link #files} if and only if files isn't empty.
|
||||
*/
|
||||
abstract Path entryPoint();
|
||||
|
||||
static ReportFiles create(ImmutableMap<Path, Supplier<byte[]>> files, Path entryPoint) {
|
||||
checkArgument(files.isEmpty() || files.containsKey(entryPoint));
|
||||
return new AutoValue_ProjectData_TaskData_ReportFiles(files, entryPoint);
|
||||
}
|
||||
|
||||
static ReportFiles createSingleFile(Path path, Supplier<byte[]> data) {
|
||||
return create(ImmutableMap.of(path, data), path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@
|
|||
package google.registry.gradle.plugin;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkNotNull;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs;
|
||||
|
|
@ -27,6 +27,7 @@ import com.google.auth.oauth2.GoogleCredentials;
|
|||
import com.google.cloud.storage.Storage;
|
||||
import com.google.cloud.storage.StorageOptions;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.Files;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
|
|
@ -38,6 +39,7 @@ import java.security.SecureRandom;
|
|||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Optional;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Supplier;
|
||||
import org.gradle.api.DefaultTask;
|
||||
import org.gradle.api.Project;
|
||||
|
|
@ -51,20 +53,32 @@ import org.gradle.api.tasks.TaskAction;
|
|||
/**
|
||||
* A task that uploads the Reports generated by other tasks to GCS.
|
||||
*/
|
||||
public class GcsReportUploader extends DefaultTask {
|
||||
public class ReportUploader extends DefaultTask {
|
||||
|
||||
private static final SecureRandom secureRandom = new SecureRandom();
|
||||
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
|
||||
private static final ImmutableMap<String, BiConsumer<ReportUploader, String>> UPLOAD_FUNCTIONS =
|
||||
ImmutableMap.of(
|
||||
"file://", ReportUploader::saveResultsToLocalFolder,
|
||||
"gcs://", ReportUploader::uploadResultsToGcs);
|
||||
|
||||
private final ArrayList<Task> tasks = new ArrayList<>();
|
||||
private final HashMap<String, StringBuilder> logs = new HashMap<>();
|
||||
private Project project;
|
||||
|
||||
private String bucket = null;
|
||||
private String destination = null;
|
||||
private String credentialsFile = null;
|
||||
private String multithreadedUpload = null;
|
||||
|
||||
public void setBucket(String bucket) {
|
||||
this.bucket = bucket;
|
||||
/** Sets the destination of the reports.
|
||||
*
|
||||
* Currently supports two types of destinations:
|
||||
*
|
||||
* - file://[absulute local path], e.g. file:///tmp/buildOutputs/
|
||||
*
|
||||
* - gcs://[bucket name]/[optional path], e.g. gcs://my-bucket/buildOutputs/
|
||||
*/
|
||||
public void setDestination(String destination) {
|
||||
this.destination = destination;
|
||||
}
|
||||
|
||||
public void setCredentialsFile(String credentialsFile) {
|
||||
|
|
@ -144,34 +158,85 @@ public class GcsReportUploader extends DefaultTask {
|
|||
: Optional.empty();
|
||||
builder
|
||||
.reportsBuilder()
|
||||
.put(type, createReportFiles(destination, entryPointHint, rootDir));
|
||||
.put(type, readFilesWithEntryPoint(destination, entryPointHint, rootDir));
|
||||
});
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private FilesWithEntryPoint generateFilesToUpload() {
|
||||
ProjectData projectData = createProjectData();
|
||||
CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData);
|
||||
return coverPageGenerator.getFilesToUpload();
|
||||
}
|
||||
|
||||
@TaskAction
|
||||
void uploadResults() {
|
||||
System.out.format("GcsReportUploader: bucket= '%s'\n", bucket);
|
||||
if (isNullOrEmpty(bucket)) {
|
||||
System.out.format("GcsReportUploader: no bucket defined. Skipping upload\n");
|
||||
System.out.format("ReportUploader: destination= '%s'\n", destination);
|
||||
|
||||
try {
|
||||
|
||||
if (isNullOrEmpty(destination)) {
|
||||
System.out.format("ReportUploader: no destination given, skipping...\n");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
uploadResultsToGcs();
|
||||
for (String key : UPLOAD_FUNCTIONS.keySet()) {
|
||||
if (destination.startsWith(key)) {
|
||||
UPLOAD_FUNCTIONS.get(key).accept(this, destination.substring(key.length()));
|
||||
return;
|
||||
}
|
||||
}
|
||||
System.out.format(
|
||||
"ReportUploader: given destination '%s' doesn't start with one of %s."
|
||||
+ " Defaulting to saving in /tmp\n",
|
||||
destination, UPLOAD_FUNCTIONS.keySet());
|
||||
saveResultsToLocalFolder("/tmp/");
|
||||
} catch (Throwable e) {
|
||||
System.out.format("GcsReportUploader: Encountered error %s\n", e);
|
||||
System.out.format("ReportUploader: Encountered error %s\n", e);
|
||||
e.printStackTrace(System.out);
|
||||
System.out.format("GcsReportUploader: skipping upload\n");
|
||||
System.out.format("ReportUploader: skipping upload\n");
|
||||
}
|
||||
}
|
||||
|
||||
void uploadResultsToGcs() {
|
||||
checkNotNull(bucket);
|
||||
ProjectData projectData = createProjectData();
|
||||
private void saveResultsToLocalFolder(String absoluteFolderName) {
|
||||
Path folder = Paths.get(absoluteFolderName, createUniqueFolderName());
|
||||
checkArgument(
|
||||
folder.isAbsolute(),
|
||||
"Local files destination must be an absolute path, but is %s",
|
||||
absoluteFolderName);
|
||||
FilesWithEntryPoint filesToUpload = generateFilesToUpload();
|
||||
System.out.format(
|
||||
"ReportUploader: going to save %s files to %s\n",
|
||||
filesToUpload.files().size(), folder);
|
||||
filesToUpload
|
||||
.files()
|
||||
.forEach((path, dataSupplier) -> saveFile(folder.resolve(path), dataSupplier));
|
||||
System.out.format(
|
||||
"ReportUploader: report saved to file://%s\n",
|
||||
folder.resolve(filesToUpload.entryPoint()));
|
||||
}
|
||||
|
||||
Path folder = Paths.get(createUniqueFolderName());
|
||||
private void saveFile(Path path, Supplier<byte[]> dataSupplier) {
|
||||
File dir = path.getParent().toFile();
|
||||
if (!dir.isDirectory()) {
|
||||
checkState(dir.mkdirs(), "Couldn't create directory %s", dir);
|
||||
}
|
||||
try {
|
||||
Files.write(dataSupplier.get(), path.toFile());
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void uploadResultsToGcs(String destination) {
|
||||
checkArgument(
|
||||
!destination.isEmpty(),
|
||||
"destination must include at least the bucket name, but is empty");
|
||||
Path bucketWithFolder = Paths.get(destination, createUniqueFolderName());
|
||||
String bucket = bucketWithFolder.getName(0).toString();
|
||||
Path folder = bucketWithFolder.subpath(1, bucketWithFolder.getNameCount());
|
||||
|
||||
StorageOptions.Builder storageOptions = StorageOptions.newBuilder();
|
||||
if (!isNullOrEmpty(credentialsFile)) {
|
||||
|
|
@ -184,26 +249,25 @@ public class GcsReportUploader extends DefaultTask {
|
|||
}
|
||||
Storage storage = storageOptions.build().getService();
|
||||
|
||||
CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData);
|
||||
ImmutableMap<Path, Supplier<byte[]>> filesToUpload = coverPageGenerator.getFilesToUpload();
|
||||
FilesWithEntryPoint filesToUpload = generateFilesToUpload();
|
||||
|
||||
System.out.format(
|
||||
"GcsReportUploader: going to upload %s files to %s/%s\n",
|
||||
filesToUpload.size(), bucket, folder);
|
||||
"ReportUploader: going to upload %s files to %s/%s\n",
|
||||
filesToUpload.files().size(), bucket, folder);
|
||||
if ("yes".equals(multithreadedUpload)) {
|
||||
System.out.format("GcsReportUploader: multi-threaded upload\n");
|
||||
uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload);
|
||||
System.out.format("ReportUploader: multi-threaded upload\n");
|
||||
uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload.files());
|
||||
} else {
|
||||
System.out.format("GcsReportUploader: single threaded upload\n");
|
||||
filesToUpload.forEach(
|
||||
System.out.format("ReportUploader: single threaded upload\n");
|
||||
filesToUpload.files().forEach(
|
||||
(path, dataSupplier) -> {
|
||||
System.out.format("GcsReportUploader: Uploading %s\n", path);
|
||||
System.out.format("ReportUploader: Uploading %s\n", path);
|
||||
uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier);
|
||||
});
|
||||
}
|
||||
System.out.format(
|
||||
"GcsReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n",
|
||||
bucket, folder.resolve(coverPageGenerator.getEntryPoint()));
|
||||
"ReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n",
|
||||
bucket, folder.resolve(filesToUpload.entryPoint()));
|
||||
}
|
||||
|
||||
void setProject(Project project) {
|
||||
|
|
@ -215,7 +279,7 @@ public class GcsReportUploader extends DefaultTask {
|
|||
}
|
||||
|
||||
private void addTask(Task task) {
|
||||
if (task instanceof GcsReportUploader) {
|
||||
if (task instanceof ReportUploader) {
|
||||
return;
|
||||
}
|
||||
tasks.add(task);
|
||||
|
|
@ -241,9 +305,9 @@ public class GcsReportUploader extends DefaultTask {
|
|||
private String createUniqueFolderName() {
|
||||
return String.format(
|
||||
"%h-%h-%h-%h",
|
||||
secureRandom.nextInt(),
|
||||
secureRandom.nextInt(),
|
||||
secureRandom.nextInt(),
|
||||
secureRandom.nextInt());
|
||||
SECURE_RANDOM.nextInt(),
|
||||
SECURE_RANDOM.nextInt(),
|
||||
SECURE_RANDOM.nextInt(),
|
||||
SECURE_RANDOM.nextInt());
|
||||
}
|
||||
}
|
||||
|
|
@ -18,20 +18,20 @@ import org.gradle.api.Plugin;
|
|||
import org.gradle.api.Project;
|
||||
|
||||
/**
|
||||
* Plugin setting up the GcsReportUploader task.
|
||||
* Plugin setting up the ReportUploader task.
|
||||
*
|
||||
* <p>It goes over all the tasks in a project and pass them on to the GcsReportUploader task for set
|
||||
* <p>It goes over all the tasks in a project and pass them on to the ReportUploader task for set
|
||||
* up.
|
||||
*
|
||||
* <p>Note that since we're passing in all the projects' tasks - this includes the GcsReportUploader
|
||||
* itself! It's up to the GcsReportuploader to take care of not having "infinite loops" caused by
|
||||
* <p>Note that since we're passing in all the projects' tasks - this includes the ReportUploader
|
||||
* itself! It's up to the ReportUploader to take care of not having "infinite loops" caused by
|
||||
* waiting for itself to end before finishing.
|
||||
*/
|
||||
public class GcsReportUploaderPlugin implements Plugin<Project> {
|
||||
public class ReportUploaderPlugin implements Plugin<Project> {
|
||||
|
||||
public void apply(Project project) {
|
||||
GcsReportUploader reportUploader =
|
||||
project.getTasks().create("gcsReportUploader", GcsReportUploader.class, task -> {
|
||||
ReportUploader reportUploader =
|
||||
project.getTasks().create("reportUploader", ReportUploader.class, task -> {
|
||||
task.setDescription("Uploads the reports to GCS bucket");
|
||||
task.setGroup("uploads");
|
||||
});
|
||||
|
|
@ -23,10 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
|
|||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.function.Supplier;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.JUnit4;
|
||||
|
|
@ -68,8 +65,8 @@ public final class CoverPageGeneratorTest {
|
|||
|
||||
private ImmutableMap<String, String> getGeneratedFiles(ProjectData project) {
|
||||
CoverPageGenerator coverPageGenerator = new CoverPageGenerator(project);
|
||||
ImmutableMap<Path, Supplier<byte[]>> files = coverPageGenerator.getFilesToUpload();
|
||||
return files.entrySet().stream()
|
||||
FilesWithEntryPoint files = coverPageGenerator.getFilesToUpload();
|
||||
return files.files().entrySet().stream()
|
||||
.collect(
|
||||
toImmutableMap(
|
||||
entry -> entry.getKey().toString(),
|
||||
|
|
@ -87,9 +84,10 @@ public final class CoverPageGeneratorTest {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testGetFilesToUpload_getEntryPoint_isIndexHtml() {
|
||||
public void testGetFilesToUpload_entryPoint_isIndexHtml() {
|
||||
CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT);
|
||||
assertThat(coverPageGenerator.getEntryPoint()).isEqualTo(Paths.get("index.html"));
|
||||
assertThat(coverPageGenerator.getFilesToUpload().entryPoint())
|
||||
.isEqualTo(Paths.get("index.html"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -220,7 +218,7 @@ public final class CoverPageGeneratorTest {
|
|||
.toBuilder()
|
||||
.putReport(
|
||||
"someReport",
|
||||
ReportFiles.create(
|
||||
FilesWithEntryPoint.create(
|
||||
ImmutableMap.of(
|
||||
Paths.get("path", "report.txt"),
|
||||
toByteArraySupplier("report content")),
|
||||
|
|
@ -242,7 +240,8 @@ public final class CoverPageGeneratorTest {
|
|||
.toBuilder()
|
||||
.putReport(
|
||||
"someReport",
|
||||
ReportFiles.create(ImmutableMap.of(), Paths.get("path", "report.txt")))
|
||||
FilesWithEntryPoint.create(
|
||||
ImmutableMap.of(), Paths.get("path", "report.txt")))
|
||||
.build())
|
||||
.build());
|
||||
assertThat(files).doesNotContainKey("path/report.txt");
|
||||
|
|
@ -263,7 +262,7 @@ public final class CoverPageGeneratorTest {
|
|||
.setLog(toByteArraySupplier("log data"))
|
||||
.putReport(
|
||||
"filledReport",
|
||||
ReportFiles.create(
|
||||
FilesWithEntryPoint.create(
|
||||
ImmutableMap.of(
|
||||
Paths.get("path-filled", "report.txt"),
|
||||
toByteArraySupplier("report content"),
|
||||
|
|
@ -272,7 +271,7 @@ public final class CoverPageGeneratorTest {
|
|||
Paths.get("path-filled", "report.txt")))
|
||||
.putReport(
|
||||
"emptyReport",
|
||||
ReportFiles.create(
|
||||
FilesWithEntryPoint.create(
|
||||
ImmutableMap.of(), Paths.get("path-empty", "report.txt")))
|
||||
.build())
|
||||
.build());
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ package google.registry.gradle.plugin;
|
|||
|
||||
import static com.google.common.collect.ImmutableMap.toImmutableMap;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.createReportFiles;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.getContentType;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath;
|
||||
import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs;
|
||||
|
|
@ -30,7 +30,6 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
|
|||
import com.google.cloud.storage.BlobInfo;
|
||||
import com.google.cloud.storage.Storage;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import google.registry.gradle.plugin.ProjectData.TaskData.ReportFiles;
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
|
@ -136,7 +135,7 @@ public final class GcsPluginUtilsTest {
|
|||
assertThat(toByteArraySupplier(file).get()).isEqualTo("some data".getBytes(UTF_8));
|
||||
}
|
||||
|
||||
private ImmutableMap<String, String> readAllFiles(ReportFiles reportFiles) {
|
||||
private ImmutableMap<String, String> readAllFiles(FilesWithEntryPoint reportFiles) {
|
||||
return reportFiles.files().entrySet().stream()
|
||||
.collect(
|
||||
toImmutableMap(
|
||||
|
|
@ -153,7 +152,8 @@ public final class GcsPluginUtilsTest {
|
|||
// Since the entry point is obvious here - any hint given is just ignored.
|
||||
File ignoredHint = folder.newFile("my/root/ignored.txt");
|
||||
|
||||
ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root);
|
||||
FilesWithEntryPoint files =
|
||||
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
|
||||
|
||||
assertThat(files.entryPoint().toString()).isEqualTo("some/path/file.txt");
|
||||
assertThat(readAllFiles(files)).containsExactly("some/path/file.txt", "some data");
|
||||
|
|
@ -168,7 +168,8 @@ public final class GcsPluginUtilsTest {
|
|||
// Since there are not files, any hint given is obvioulsy wrong and will be ignored.
|
||||
File ignoredHint = folder.newFile("my/root/ignored.txt");
|
||||
|
||||
ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root);
|
||||
FilesWithEntryPoint files =
|
||||
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
|
||||
|
||||
assertThat(files.entryPoint().toString()).isEqualTo("non/existing.txt");
|
||||
assertThat(files.files()).isEmpty();
|
||||
|
|
@ -183,7 +184,8 @@ public final class GcsPluginUtilsTest {
|
|||
// Since there are not files, any hint given is obvioulsy wrong and will be ignored.
|
||||
File ignoredHint = folder.newFile("my/root/ignored.txt");
|
||||
|
||||
ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root);
|
||||
FilesWithEntryPoint files =
|
||||
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
|
||||
|
||||
assertThat(files.entryPoint().toString()).isEqualTo("some/path");
|
||||
assertThat(files.files()).isEmpty();
|
||||
|
|
@ -200,7 +202,8 @@ public final class GcsPluginUtilsTest {
|
|||
// Since the entry point is obvious here - any hint given is just ignored.
|
||||
File ignoredHint = folder.newFile("my/root/ignored.txt");
|
||||
|
||||
ReportFiles files = createReportFiles(destination, Optional.of(ignoredHint), root);
|
||||
FilesWithEntryPoint files =
|
||||
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
|
||||
|
||||
assertThat(files.entryPoint().toString()).isEqualTo("some/path/a/file.txt");
|
||||
assertThat(readAllFiles(files)).containsExactly("some/path/a/file.txt", "some data");
|
||||
|
|
@ -227,7 +230,7 @@ public final class GcsPluginUtilsTest {
|
|||
Files.write(
|
||||
folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
|
||||
|
||||
ReportFiles files = createReportFiles(destination, Optional.empty(), root);
|
||||
FilesWithEntryPoint files = readFilesWithEntryPoint(destination, Optional.empty(), root);
|
||||
|
||||
assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip");
|
||||
assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip");
|
||||
|
|
@ -255,7 +258,8 @@ public final class GcsPluginUtilsTest {
|
|||
Files.write(
|
||||
folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
|
||||
|
||||
ReportFiles files = createReportFiles(destination, Optional.of(badEntryPoint), root);
|
||||
FilesWithEntryPoint files =
|
||||
readFilesWithEntryPoint(destination, Optional.of(badEntryPoint), root);
|
||||
|
||||
assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip");
|
||||
assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip");
|
||||
|
|
@ -278,7 +282,8 @@ public final class GcsPluginUtilsTest {
|
|||
Files.write(
|
||||
folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
|
||||
|
||||
ReportFiles files = createReportFiles(destination, Optional.of(goodEntryPoint), root);
|
||||
FilesWithEntryPoint files =
|
||||
readFilesWithEntryPoint(destination, Optional.of(goodEntryPoint), root);
|
||||
|
||||
assertThat(files.entryPoint().toString()).isEqualTo("some/path/index.html");
|
||||
assertThat(readAllFiles(files))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
repositoryUrl=
|
||||
publishUrl=
|
||||
gcsBucket=
|
||||
gcsCredentialsFile=
|
||||
gcsMultithreadedUpload=
|
||||
uploaderDestination=
|
||||
uploaderCredentialsFile=
|
||||
uploaderMultithreadedUpload=
|
||||
mavenCoordinateFile=
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue