Refactor to be more in line with a standard Gradle project structure

This commit is contained in:
Gus Brodman 2019-05-21 14:12:47 -04:00
parent 8fa45e8c76
commit a7a983bfed
3141 changed files with 99 additions and 100 deletions

View file

@ -0,0 +1,198 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
import static com.google.common.io.Resources.getResource;
import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.template.soy.SoyFileSet;
import com.google.template.soy.tofu.SoyTofu;
import google.registry.gradle.plugin.ProjectData.TaskData;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.function.Supplier;
import java.util.stream.Collectors;
/**
* Creates the files for a web-page summary of a given {@Link ProjectData}.
*
* <p>The main job of this class is rendering a tailored cover page that includes information about
* the project and any task that ran.
*
* <p>It returns all the files that need uploading for the cover page to work. This includes any
* report and log files linked to in the ProjectData, as well as a cover page (and associated
* resources such as CSS files).
*/
final class CoverPageGenerator {
/** List of all resource files that will be uploaded as-is. */
private static final ImmutableSet<Path> STATIC_RESOURCE_FILES =
ImmutableSet.of(Paths.get("css", "style.css"));
/** Name of the entry-point file that will be created. */
private static final Path ENTRY_POINT = Paths.get("index.html");
private final ProjectData projectData;
private final ImmutableSetMultimap<TaskData.State, TaskData> tasksByState;
/**
* The compiled SOY files.
*
* <p>Will be generated only when actually needed, because it takes a while to compile and we
* don't want that to happen unless we actually use it.
*/
private SoyTofu tofu = null;
CoverPageGenerator(ProjectData projectData) {
this.projectData = projectData;
this.tasksByState =
projectData.tasks().stream().collect(toImmutableSetMultimap(TaskData::state, task -> task));
}
/**
* Returns all the files that need uploading for the cover page to work.
*
* <p>This includes all the report files as well, to make sure that the link works.
*/
FilesWithEntryPoint getFilesToUpload() {
ImmutableMap.Builder<Path, Supplier<byte[]>> builder = new ImmutableMap.Builder<>();
// Add all the static resource pages
STATIC_RESOURCE_FILES.stream().forEach(file -> builder.put(file, resourceLoader(file)));
// Create the cover page
// Note that the ByteArraySupplier here is lazy - the createCoverPage function is only called
// when the resulting Supplier's get function is called.
builder.put(ENTRY_POINT, toByteArraySupplier(this::createCoverPage));
// Add all the files from the tasks
tasksByState.values().stream()
.flatMap(task -> task.reports().values().stream())
.forEach(reportFiles -> builder.putAll(reportFiles.files()));
// Add the logs of every test
tasksByState.values().stream()
.filter(task -> task.log().isPresent())
.forEach(task -> builder.put(getLogPath(task), task.log().get()));
return FilesWithEntryPoint.create(builder.build(), ENTRY_POINT);
}
/** Renders the cover page. */
private String createCoverPage() {
return getTofu()
.newRenderer("google.registry.gradle.plugin.coverPage")
.setData(getSoyData())
.render();
}
/** Converts the projectData and all taskData into all the data the soy template needs. */
private ImmutableMap<String, Object> getSoyData() {
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
TaskData.State state =
tasksByState.containsKey(TaskData.State.FAILURE)
? TaskData.State.FAILURE
: TaskData.State.SUCCESS;
String title =
state != TaskData.State.FAILURE
? "Success!"
: "Failed: "
+ tasksByState.get(state).stream()
.map(TaskData::uniqueName)
.collect(Collectors.joining(", "));
builder.put("projectState", state.toString());
builder.put("title", title);
builder.put("cssFiles", ImmutableSet.of("css/style.css"));
builder.put("invocation", getInvocation());
builder.put("tasksByState", getTasksByStateSoyData());
return builder.build();
}
/**
* Returns a soy-friendly map from the TaskData.State to the task itslef.
*
* <p>The key order in the resulting map is always the same (the order from the enum definition)
* no matter the key order in the original tasksByState map.
*/
private ImmutableMap<String, Object> getTasksByStateSoyData() {
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
// We go over the States in the order they are defined rather than the order in which they
// happen to be in the tasksByState Map.
//
// That way we guarantee a consistent order.
for (TaskData.State state : TaskData.State.values()) {
builder.put(
state.toString(),
tasksByState.get(state).stream()
.map(task -> taskDataToSoy(task))
.collect(toImmutableList()));
}
return builder.build();
}
/** returns a soy-friendly version of the given task data. */
static ImmutableMap<String, Object> taskDataToSoy(TaskData task) {
return new ImmutableMap.Builder<String, Object>()
.put("uniqueName", task.uniqueName())
.put("description", task.description())
.put("log", task.log().isPresent() ? getLogPath(task).toString() : "")
.put(
"reports",
task.reports().entrySet().stream()
.collect(
toImmutableMap(
entry -> entry.getKey(),
entry ->
entry.getValue().files().isEmpty()
? ""
: entry.getValue().entryPoint().toString())))
.build();
}
private String getInvocation() {
StringBuilder builder = new StringBuilder();
builder.append("./gradlew");
projectData.tasksRequested().forEach(task -> builder.append(" ").append(task));
projectData
.projectProperties()
.forEach((key, value) -> builder.append(String.format(" -P %s=%s", key, value)));
return builder.toString();
}
/** Returns a lazily created soy renderer */
private SoyTofu getTofu() {
if (tofu == null) {
tofu =
SoyFileSet.builder()
.add(getResource(CoverPageGenerator.class, "soy/coverpage.soy"))
.build()
.compileToTofu();
}
return tofu;
}
private static Path getLogPath(TaskData task) {
// TODO(guyben):escape uniqueName to guarantee legal file name.
return Paths.get("logs", task.uniqueName() + ".log");
}
private static Supplier<byte[]> resourceLoader(Path path) {
return toByteArraySupplier(getResource(CoverPageGenerator.class, path.toString()));
}
}

View file

@ -0,0 +1,59 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import static com.google.common.base.Preconditions.checkArgument;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableMap;
import java.nio.file.Path;
import java.util.function.Supplier;
/**
* Holds a set of files with a browser-friendly entry point to those files.
*
* <p>The file data is lazily generated.
*
* <p>If there is at least one file, it's guaranteed that the entry point is one of these files.
*/
@AutoValue
abstract class FilesWithEntryPoint {
/**
* All files that are part of this report, keyed from their path to a supplier of their content.
*
* <p>The reason we use a supplier instead of loading the content is in case the content is very
* large...
*
* <p>Also, no point in doing IO before we need it!
*/
abstract ImmutableMap<Path, Supplier<byte[]>> files();
/**
* The file that gives access (links...) to all the data in the report.
*
* <p>Guaranteed to be a key in {@link #files} if and only if files isn't empty.
*/
abstract Path entryPoint();
static FilesWithEntryPoint create(ImmutableMap<Path, Supplier<byte[]>> files, Path entryPoint) {
checkArgument(files.isEmpty() || files.containsKey(entryPoint));
return new AutoValue_FilesWithEntryPoint(files, entryPoint);
}
static FilesWithEntryPoint createSingleFile(Path path, Supplier<byte[]> data) {
return create(ImmutableMap.of(path, data), path);
}
}

View file

@ -0,0 +1,216 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.Iterables.getOnlyElement;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.URL;
import java.nio.file.Path;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/** Utility functions used in the GCS plugin. */
final class GcsPluginUtils {
private static final ImmutableMap<String, String> EXTENSION_TO_CONTENT_TYPE =
new ImmutableMap.Builder<String, String>()
.put("html", "text/html")
.put("htm", "text/html")
.put("log", "text/plain")
.put("txt", "text/plain")
.put("css", "text/css")
.put("xml", "text/xml")
.put("zip", "application/zip")
.put("js", "text/javascript")
.build();
private static final String DEFAULT_CONTENT_TYPE = "application/octet-stream";
static Path toNormalizedPath(File file) {
return file.toPath().toAbsolutePath().normalize();
}
static String getContentType(String fileName) {
return EXTENSION_TO_CONTENT_TYPE.getOrDefault(
Files.getFileExtension(fileName), DEFAULT_CONTENT_TYPE);
}
static void uploadFileToGcs(
Storage storage, String bucket, Path path, Supplier<byte[]> dataSupplier) {
String filename = path.toString();
storage.create(
BlobInfo.newBuilder(bucket, filename).setContentType(getContentType(filename)).build(),
dataSupplier.get());
}
static void uploadFilesToGcsMultithread(
Storage storage, String bucket, Path folder, Map<Path, Supplier<byte[]>> files) {
ImmutableMap.Builder<Path, Thread> threads = new ImmutableMap.Builder<>();
files.forEach(
(path, dataSupplier) -> {
Thread thread =
new Thread(
() -> uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier));
thread.start();
threads.put(path, thread);
});
threads
.build()
.forEach(
(path, thread) -> {
try {
thread.join();
} catch (InterruptedException e) {
System.out.format("Upload of %s interrupted", path);
}
});
}
static Supplier<byte[]> toByteArraySupplier(String data) {
return () -> data.getBytes(UTF_8);
}
static Supplier<byte[]> toByteArraySupplier(Supplier<String> dataSupplier) {
return () -> dataSupplier.get().getBytes(UTF_8);
}
static Supplier<byte[]> toByteArraySupplier(File file) {
return () -> {
try {
return Files.toByteArray(file);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
}
static Supplier<byte[]> toByteArraySupplier(URL url) {
return () -> {
try {
return Resources.toByteArray(url);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
}
/**
* Reads all the files generated by a Report into a FilesWithEntryPoint object.
*
* <p>Every FilesWithEntryPoint must have a single link "entry point" that gives users access to
* all the files. If the report generated just one file - we will just link to that file.
*
* <p>However, if the report generated more than one file - the only thing we can safely do is to
* zip all the files and link to the zip file.
*
* <p>As an alternative to using a zip file, we allow the caller to supply an optional "entry
* point" file that will link to all the other files. If that file is given and is "appropriate"
* (exists and is in the correct location) - we will upload all the report files "as is" and link
* to the entry file.
*
* @param destination the location of the output. Either a file or a directory. If a directory -
* then all the files inside that directory are the outputs we're looking for.
* @param entryPointHint If present - a hint to what the entry point to this directory tree is.
* Will only be used if all of the following apply: (a) {@code
* destination.isDirectory()==true}, (b) there are 2 or more files in the {@code destination}
* directory, and (c) {@code entryPointHint.get()} is one of the files nested inside of the
* {@code destination} directory.
*/
static FilesWithEntryPoint readFilesWithEntryPoint(
File destination, Optional<File> entryPointHint, Path rootDir) {
Path destinationPath = rootDir.relativize(toNormalizedPath(destination));
if (destination.isFile()) {
// The destination is a single file - find its root, and add this single file to the
// FilesWithEntryPoint.
return FilesWithEntryPoint.createSingleFile(
destinationPath, toByteArraySupplier(destination));
}
if (!destination.isDirectory()) {
// This isn't a file nor a directory - so it doesn't exist! Return empty FilesWithEntryPoint
return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath);
}
// The destination is a directory - find all the actual files first
ImmutableMap<Path, Supplier<byte[]>> files =
Streams.stream(Files.fileTraverser().depthFirstPreOrder(destination))
.filter(File::isFile)
.collect(
toImmutableMap(
file -> rootDir.relativize(toNormalizedPath(file)),
file -> toByteArraySupplier(file)));
if (files.isEmpty()) {
// The directory exists, but is empty. Return empty FilesWithEntryPoint
return FilesWithEntryPoint.create(ImmutableMap.of(), destinationPath);
}
if (files.size() == 1) {
// We got a directory, but it only has a single file. We can link to that.
return FilesWithEntryPoint.create(files, getOnlyElement(files.keySet()));
}
// There are multiple files in the report! We need to check the entryPointHint
Optional<Path> entryPointPath =
entryPointHint.map(file -> rootDir.relativize(toNormalizedPath(file)));
if (entryPointPath.isPresent() && files.containsKey(entryPointPath.get())) {
// We were given the entry point! Use it!
return FilesWithEntryPoint.create(files, entryPointPath.get());
}
// We weren't given an appropriate entry point. But we still need a single link to all this data
// - so we'll zip it and just host a single file.
Path zipFilePath = destinationPath.resolve(destinationPath.getFileName().toString() + ".zip");
return FilesWithEntryPoint.createSingleFile(zipFilePath, createZippedByteArraySupplier(files));
}
static Supplier<byte[]> createZippedByteArraySupplier(Map<Path, Supplier<byte[]>> files) {
return () -> zipFiles(files);
}
private static byte[] zipFiles(Map<Path, Supplier<byte[]>> files) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
try (ZipOutputStream zip = new ZipOutputStream(output)) {
for (Path path : files.keySet()) {
zip.putNextEntry(new ZipEntry(path.toString()));
zip.write(files.get(path).get());
zip.closeEntry();
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return output.toByteArray();
}
private GcsPluginUtils() {}
}

View file

@ -0,0 +1,138 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import google.registry.gradle.plugin.ProjectData.TaskData;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
/**
* All the data of a root Gradle project.
*
* <p>This is basically all the "relevant" data from a Gradle Project, arranged in an immutable and
* more convenient way.
*/
@AutoValue
abstract class ProjectData {
abstract String name();
abstract String description();
abstract String gradleVersion();
abstract ImmutableMap<String, String> projectProperties();
abstract ImmutableMap<String, String> systemProperties();
abstract ImmutableSet<String> tasksRequested();
abstract ImmutableSet<TaskData> tasks();
abstract Builder toBuilder();
static Builder builder() {
return new AutoValue_ProjectData.Builder();
}
@AutoValue.Builder
abstract static class Builder {
abstract Builder setName(String name);
abstract Builder setDescription(String description);
abstract Builder setGradleVersion(String gradleVersion);
abstract Builder setProjectProperties(Map<String, String> projectProperties);
abstract Builder setSystemProperties(Map<String, String> systemProperties);
abstract Builder setTasksRequested(Iterable<String> tasksRequested);
abstract ImmutableSet.Builder<TaskData> tasksBuilder();
Builder addTask(TaskData task) {
tasksBuilder().add(task);
return this;
}
abstract ProjectData build();
}
/**
* Relevant data to a single Task's.
*
* <p>Some Tasks are also "Reporting", meaning they create file outputs we want to upload in
* various formats. The format that interests us the most is "html", as that's nicely browsable,
* but they might also have other formats.
*/
@AutoValue
abstract static class TaskData {
enum State {
/** The task has failed for some reason. */
FAILURE,
/** The task was actually run and has finished successfully. */
SUCCESS,
/** The task was up-to-date and successful, and hence didn't need to run again. */
UP_TO_DATE;
}
abstract String uniqueName();
abstract String description();
abstract State state();
abstract Optional<Supplier<byte[]>> log();
/**
* Returns the FilesWithEntryPoint for every report, keyed on the report type.
*
* <p>The "html" report type is the most interesting, but there are other report formats.
*/
abstract ImmutableMap<String, FilesWithEntryPoint> reports();
abstract Builder toBuilder();
static Builder builder() {
return new AutoValue_ProjectData_TaskData.Builder();
}
@AutoValue.Builder
abstract static class Builder {
abstract Builder setUniqueName(String name);
abstract Builder setDescription(String description);
abstract Builder setState(State state);
abstract Builder setLog(Supplier<byte[]> log);
abstract ImmutableMap.Builder<String, FilesWithEntryPoint> reportsBuilder();
Builder putReport(String type, FilesWithEntryPoint reportFiles) {
reportsBuilder().put(type, reportFiles);
return this;
}
abstract TaskData build();
}
}
}

View file

@ -0,0 +1,311 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint;
import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath;
import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs;
import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;
import google.registry.gradle.plugin.ProjectData.TaskData;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Supplier;
import org.gradle.api.DefaultTask;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.reporting.DirectoryReport;
import org.gradle.api.reporting.Report;
import org.gradle.api.reporting.ReportContainer;
import org.gradle.api.reporting.Reporting;
import org.gradle.api.tasks.TaskAction;
/** A task that uploads the Reports generated by other tasks to GCS. */
public class ReportUploader extends DefaultTask {
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
private static final ImmutableMap<String, BiConsumer<ReportUploader, String>> UPLOAD_FUNCTIONS =
ImmutableMap.of(
"file://", ReportUploader::saveResultsToLocalFolder,
"gcs://", ReportUploader::uploadResultsToGcs);
private final ArrayList<Task> tasks = new ArrayList<>();
private final HashMap<String, StringBuilder> logs = new HashMap<>();
private Project project;
private String destination = null;
private String credentialsFile = null;
private String multithreadedUpload = null;
/**
* Sets the destination of the reports.
*
* <p>Currently supports two types of destinations:
*
* <ul>
* <li>file://[absulute local path], e.g. file:///tmp/buildOutputs/
* <li>gcs://[bucket name]/[optional path], e.g. gcs://my-bucket/buildOutputs/
* </ul>
*/
public void setDestination(String destination) {
this.destination = destination;
}
public void setCredentialsFile(String credentialsFile) {
this.credentialsFile = credentialsFile;
}
public void setMultithreadedUpload(String multithreadedUpload) {
this.multithreadedUpload = multithreadedUpload;
}
/** Converts the given Gradle Project into a ProjectData. */
private ProjectData createProjectData() {
ProjectData.Builder builder =
ProjectData.builder()
.setName(project.getPath() + project.getName())
.setDescription(
Optional.ofNullable(project.getDescription()).orElse("[No description available]"))
.setGradleVersion(project.getGradle().getGradleVersion())
.setProjectProperties(project.getGradle().getStartParameter().getProjectProperties())
.setSystemProperties(project.getGradle().getStartParameter().getSystemPropertiesArgs())
.setTasksRequested(project.getGradle().getStartParameter().getTaskNames());
Path rootDir = toNormalizedPath(project.getRootDir());
tasks.stream()
.filter(task -> task.getState().getExecuted() || task.getState().getUpToDate())
.map(task -> createTaskData(task, rootDir))
.forEach(builder.tasksBuilder()::add);
return builder.build();
}
/**
* Converts a Gradle Task into a TaskData.
*
* @param rootDir the root directory of the main Project - used to get the relative path of any
* Task files.
*/
private TaskData createTaskData(Task task, Path rootDir) {
TaskData.State state =
task.getState().getFailure() != null
? TaskData.State.FAILURE
: task.getState().getUpToDate() ? TaskData.State.UP_TO_DATE : TaskData.State.SUCCESS;
String log = logs.get(task.getPath()).toString();
TaskData.Builder builder =
TaskData.builder()
.setState(state)
.setUniqueName(task.getPath())
.setDescription(
Optional.ofNullable(task.getDescription()).orElse("[No description available]"));
if (!log.isEmpty()) {
builder.setLog(toByteArraySupplier(log));
}
Reporting<? extends ReportContainer<? extends Report>> reporting = asReporting(task);
if (reporting != null) {
// This Task is also a Reporting task! It has a destination file/directory for every supported
// format.
// Add the files for each of the formats into the ReportData.
reporting
.getReports()
.getAsMap()
.forEach(
(type, report) -> {
File destination = report.getDestination();
// The destination could be a file, or a directory. If it's a directory - the Report
// could have created multiple files - and we need to know to which one of those to
// link.
//
// If we're lucky, whoever implemented the Report made sure to extend
// DirectoryReport, which gives us the entry point to all the files.
//
// This isn't guaranteed though, as it depends on the implementer.
Optional<File> entryPointHint =
destination.isDirectory() && (report instanceof DirectoryReport)
? Optional.ofNullable(((DirectoryReport) report).getEntryPoint())
: Optional.empty();
builder
.reportsBuilder()
.put(type, readFilesWithEntryPoint(destination, entryPointHint, rootDir));
});
}
return builder.build();
}
private FilesWithEntryPoint generateFilesToUpload() {
ProjectData projectData = createProjectData();
CoverPageGenerator coverPageGenerator = new CoverPageGenerator(projectData);
return coverPageGenerator.getFilesToUpload();
}
@TaskAction
void uploadResults() {
System.out.format("ReportUploader: destination= '%s'\n", destination);
try {
if (isNullOrEmpty(destination)) {
System.out.format("ReportUploader: no destination given, skipping...\n");
return;
}
for (String key : UPLOAD_FUNCTIONS.keySet()) {
if (destination.startsWith(key)) {
UPLOAD_FUNCTIONS.get(key).accept(this, destination.substring(key.length()));
return;
}
}
System.out.format(
"ReportUploader: given destination '%s' doesn't start with one of %s."
+ " Defaulting to saving in /tmp\n",
destination, UPLOAD_FUNCTIONS.keySet());
saveResultsToLocalFolder("/tmp/");
} catch (Throwable e) {
System.out.format("ReportUploader: Encountered error %s\n", e);
e.printStackTrace(System.out);
System.out.format("ReportUploader: skipping upload\n");
}
}
private void saveResultsToLocalFolder(String absoluteFolderName) {
Path folder = Paths.get(absoluteFolderName, createUniqueFolderName());
checkArgument(
folder.isAbsolute(),
"Local files destination must be an absolute path, but is %s",
absoluteFolderName);
FilesWithEntryPoint filesToUpload = generateFilesToUpload();
System.out.format(
"ReportUploader: going to save %s files to %s\n", filesToUpload.files().size(), folder);
filesToUpload
.files()
.forEach((path, dataSupplier) -> saveFile(folder.resolve(path), dataSupplier));
System.out.format(
"ReportUploader: report saved to file://%s\n", folder.resolve(filesToUpload.entryPoint()));
}
private void saveFile(Path path, Supplier<byte[]> dataSupplier) {
File dir = path.getParent().toFile();
if (!dir.isDirectory()) {
checkState(dir.mkdirs(), "Couldn't create directory %s", dir);
}
try {
Files.write(dataSupplier.get(), path.toFile());
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
private void uploadResultsToGcs(String destination) {
checkArgument(
!destination.isEmpty(), "destination must include at least the bucket name, but is empty");
Path bucketWithFolder = Paths.get(destination, createUniqueFolderName());
String bucket = bucketWithFolder.getName(0).toString();
Path folder = bucketWithFolder.subpath(1, bucketWithFolder.getNameCount());
StorageOptions.Builder storageOptions = StorageOptions.newBuilder();
if (!isNullOrEmpty(credentialsFile)) {
try {
storageOptions.setCredentials(
GoogleCredentials.fromStream(new FileInputStream(credentialsFile)));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
Storage storage = storageOptions.build().getService();
FilesWithEntryPoint filesToUpload = generateFilesToUpload();
System.out.format(
"ReportUploader: going to upload %s files to %s/%s\n",
filesToUpload.files().size(), bucket, folder);
if ("yes".equals(multithreadedUpload)) {
System.out.format("ReportUploader: multi-threaded upload\n");
uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload.files());
} else {
System.out.format("ReportUploader: single threaded upload\n");
filesToUpload
.files()
.forEach(
(path, dataSupplier) -> {
System.out.format("ReportUploader: Uploading %s\n", path);
uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier);
});
}
System.out.format(
"ReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n",
bucket, folder.resolve(filesToUpload.entryPoint()));
}
void setProject(Project project) {
this.project = project;
for (Project subProject : project.getAllprojects()) {
subProject.getTasks().all(this::addTask);
}
}
private void addTask(Task task) {
if (task instanceof ReportUploader) {
return;
}
tasks.add(task);
StringBuilder log = new StringBuilder();
checkArgument(
!logs.containsKey(task.getPath()),
"Multiple tasks with the same .getPath()=%s",
task.getPath());
logs.put(task.getPath(), log);
task.getLogging().addStandardOutputListener(output -> log.append(output));
task.getLogging().addStandardErrorListener(output -> log.append(output));
task.finalizedBy(this);
}
@SuppressWarnings("unchecked")
private static Reporting<? extends ReportContainer<? extends Report>> asReporting(Task task) {
if (task instanceof Reporting) {
return (Reporting<? extends ReportContainer<? extends Report>>) task;
}
return null;
}
private String createUniqueFolderName() {
return String.format(
"%h-%h-%h-%h",
SECURE_RANDOM.nextInt(),
SECURE_RANDOM.nextInt(),
SECURE_RANDOM.nextInt(),
SECURE_RANDOM.nextInt());
}
}

View file

@ -0,0 +1,47 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
/**
* Plugin setting up the ReportUploader task.
*
* <p>It goes over all the tasks in a project and pass them on to the ReportUploader task for set
* up.
*
* <p>Note that since we're passing in all the projects' tasks - this includes the ReportUploader
* itself! It's up to the ReportUploader to take care of not having "infinite loops" caused by
* waiting for itself to end before finishing.
*/
public class ReportUploaderPlugin implements Plugin<Project> {
@Override
public void apply(Project project) {
ReportUploader reportUploader =
project
.getTasks()
.create(
"reportUploader",
ReportUploader.class,
task -> {
task.setDescription("Uploads the reports to GCS bucket");
task.setGroup("uploads");
});
reportUploader.setProject(project);
}
}

View file

@ -0,0 +1,27 @@
body {
font-family: sans-serif;
}
.task_state_SUCCESS {
color: green;
}
.task_state_FAILURE {
color: red;
}
.task_name {
display: block;
font-size: larger;
font-weight: bold;
}
.task_description {
display: block;
margin-left: 1em;
color: gray;
}
.report_links {
margin-left: 1em;
}
.report_link_broken {
text-decoration: line-through;
color: gray;
}

View file

@ -0,0 +1,107 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
{namespace google.registry.gradle.plugin}
{template .coverPage}
{@param title: string}
{@param cssFiles: list<string>}
{@param projectState: string}
{@param invocation: string}
{@param tasksByState: map<string, list<[uniqueName: string, description: string, log: string, reports: map<string, string>]>>}
<title>{$title}</title>
{for $cssFile in $cssFiles}
<link rel="stylesheet" type="text/css" href="{$cssFile}">
{/for}
<body>
<div class="project">
<h1 class="project_title {'task_state_' + $projectState}">{$title}</h1>
<span class="project_subtitle">
Build results for <span class="project_invocation">{$invocation}</span>
</span>
{for $taskState in mapKeys($tasksByState)}
{if length($tasksByState[$taskState]) > 0}
{call .tasksOfState}
{param state: $taskState /}
{param tasks: $tasksByState[$taskState] /}
{/call}
{/if}
{/for}
</div>
</body>
{/template}
{template .tasksOfState}
{@param state: string}
{@param tasks: list<[uniqueName: string, description: string, log: string, reports: map<string, string>]>}
<div class="{'task_state_' + $state}">
<p>{$state}</p>
// Place the tasks with actual reports first, since those are more likely to be useful
{for $task in $tasks}
{if length(mapKeys($task.reports)) > 0}
{call .task}
{param task: $task /}
{/call}
{/if}
{/for}
// Followup with reports without links
{for $task in $tasks}
{if length(mapKeys($task.reports)) == 0}
{call .task}
{param task: $task /}
{/call}
{/if}
{/for}
</div>
{/template}
{template .task}
{@param task: [uniqueName: string, description: string, log: string, reports: map<string, string>]}
{call .taskInternal}
{param uniqueName: $task.uniqueName /}
{param description: $task.description /}
{param log: $task.log /}
{param reports: $task.reports /}
{/call}
{/template}
{template .taskInternal}
{@param uniqueName: string}
{@param description: string}
{@param log: string}
{@param reports: map<string, string>}
<div class="task">
<span class="task_name">{$uniqueName}</span>
<span class="task_description">{$description}</span>
<span class="report_links">
{if $log}
<a href="{$log}">[log]</a>
{else}
<span class="report_link_broken">[log]</span>
{/if}
{for $type in mapKeys($reports)}
{if $reports[$type]}
<a href="{$reports[$type]}">[{$type}]</a>
{else}
<span class="report_link_broken">[{$type}]</span>
{/if}
{/for}
</span>
</div>
{/template}

View file

@ -0,0 +1,287 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import google.registry.gradle.plugin.ProjectData.TaskData;
import java.nio.file.Paths;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Tests for {@link CoverPageGenerator} */
@RunWith(JUnit4.class)
public final class CoverPageGeneratorTest {
private static final ProjectData EMPTY_PROJECT =
ProjectData.builder()
.setName("project-name")
.setDescription("project-description")
.setGradleVersion("gradle-version")
.setProjectProperties(ImmutableMap.of("key", "value"))
.setSystemProperties(ImmutableMap.of())
.setTasksRequested(ImmutableSet.of(":a:task1", ":a:task2"))
.build();
private static final TaskData EMPTY_TASK_SUCCESS =
TaskData.builder()
.setUniqueName("task-success")
.setDescription("a successful task")
.setState(TaskData.State.SUCCESS)
.build();
private static final TaskData EMPTY_TASK_FAILURE =
TaskData.builder()
.setUniqueName("task-failure")
.setDescription("a failed task")
.setState(TaskData.State.FAILURE)
.build();
private static final TaskData EMPTY_TASK_UP_TO_DATE =
TaskData.builder()
.setUniqueName("task-up-to-date")
.setDescription("an up-to-date task")
.setState(TaskData.State.UP_TO_DATE)
.build();
private ImmutableMap<String, String> getGeneratedFiles(ProjectData project) {
CoverPageGenerator coverPageGenerator = new CoverPageGenerator(project);
FilesWithEntryPoint files = coverPageGenerator.getFilesToUpload();
return files.files().entrySet().stream()
.collect(
toImmutableMap(
entry -> entry.getKey().toString(),
entry -> new String(entry.getValue().get(), UTF_8)));
}
private String getContentOfGeneratedFile(ProjectData project, String expectedPath) {
ImmutableMap<String, String> files = getGeneratedFiles(project);
assertThat(files).containsKey(expectedPath);
return files.get(expectedPath);
}
private String getCoverPage(ProjectData project) {
return getContentOfGeneratedFile(project, "index.html");
}
@Test
public void testGetFilesToUpload_entryPoint_isIndexHtml() {
CoverPageGenerator coverPageGenerator = new CoverPageGenerator(EMPTY_PROJECT);
assertThat(coverPageGenerator.getFilesToUpload().entryPoint())
.isEqualTo(Paths.get("index.html"));
}
@Test
public void testGetFilesToUpload_containsEntryFile() {
String content = getContentOfGeneratedFile(EMPTY_PROJECT, "index.html");
assertThat(content)
.contains(
"<span class=\"project_invocation\">./gradlew :a:task1 :a:task2 -P key=value</span>");
}
@Test
public void testCoverPage_showsFailedTask() {
String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_FAILURE).build());
assertThat(content).contains("task-failure");
assertThat(content).contains("<p>FAILURE</p>");
assertThat(content).doesNotContain("<p>SUCCESS</p>");
assertThat(content).doesNotContain("<p>UP_TO_DATE</p>");
}
@Test
public void testCoverPage_showsSuccessfulTask() {
String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_SUCCESS).build());
assertThat(content).contains("task-success");
assertThat(content).doesNotContain("<p>FAILURE</p>");
assertThat(content).contains("<p>SUCCESS</p>");
assertThat(content).doesNotContain("<p>UP_TO_DATE</p>");
}
@Test
public void testCoverPage_showsUpToDateTask() {
String content = getCoverPage(EMPTY_PROJECT.toBuilder().addTask(EMPTY_TASK_UP_TO_DATE).build());
assertThat(content).contains("task-up-to-date");
assertThat(content).doesNotContain("<p>FAILURE</p>");
assertThat(content).doesNotContain("<p>SUCCESS</p>");
assertThat(content).contains("<p>UP_TO_DATE</p>");
}
@Test
public void testCoverPage_failedAreFirst() {
String content =
getCoverPage(
EMPTY_PROJECT
.toBuilder()
.addTask(EMPTY_TASK_UP_TO_DATE)
.addTask(EMPTY_TASK_FAILURE)
.addTask(EMPTY_TASK_SUCCESS)
.build());
assertThat(content).contains("<p>FAILURE</p>");
assertThat(content).contains("<p>SUCCESS</p>");
assertThat(content).contains("<p>UP_TO_DATE</p>");
assertThat(content).containsMatch("(?s)<p>FAILURE</p>.*<p>SUCCESS</p>");
assertThat(content).containsMatch("(?s)<p>FAILURE</p>.*<p>UP_TO_DATE</p>");
assertThat(content).doesNotContainMatch("(?s)<p>SUCCESS</p>.*<p>FAILURE</p>");
assertThat(content).doesNotContainMatch("(?s)<p>UP_TO_DATE</p>.*<p>FAILURE</p>");
}
@Test
public void testCoverPage_failingTask_statusIsFailure() {
String content =
getCoverPage(
EMPTY_PROJECT
.toBuilder()
.addTask(EMPTY_TASK_UP_TO_DATE)
.addTask(EMPTY_TASK_FAILURE)
.addTask(EMPTY_TASK_SUCCESS)
.build());
assertThat(content).contains("<title>Failed: task-failure</title>");
}
@Test
public void testCoverPage_noFailingTask_statusIsSuccess() {
String content =
getCoverPage(
EMPTY_PROJECT
.toBuilder()
.addTask(EMPTY_TASK_UP_TO_DATE)
.addTask(EMPTY_TASK_SUCCESS)
.build());
assertThat(content).contains("<title>Success!</title>");
}
@Test
public void testGetFilesToUpload_containsCssFile() {
ImmutableMap<String, String> files = getGeneratedFiles(EMPTY_PROJECT);
assertThat(files).containsKey("css/style.css");
assertThat(files.get("css/style.css")).contains("body {");
assertThat(files.get("index.html"))
.contains("<link rel=\"stylesheet\" type=\"text/css\" href=\"css/style.css\">");
}
@Test
public void testCreateReportFiles_taskWithLog() {
ImmutableMap<String, String> files =
getGeneratedFiles(
EMPTY_PROJECT
.toBuilder()
.addTask(
EMPTY_TASK_SUCCESS
.toBuilder()
.setUniqueName("my:name")
.setLog(toByteArraySupplier("my log data"))
.build())
.build());
assertThat(files).containsEntry("logs/my:name.log", "my log data");
assertThat(files.get("index.html")).contains("<a href=\"logs/my:name.log\">[log]</a>");
}
@Test
public void testCreateReportFiles_taskWithoutLog() {
ImmutableMap<String, String> files =
getGeneratedFiles(
EMPTY_PROJECT
.toBuilder()
.addTask(EMPTY_TASK_SUCCESS.toBuilder().setUniqueName("my:name").build())
.build());
assertThat(files).doesNotContainKey("logs/my:name.log");
assertThat(files.get("index.html")).contains("<span class=\"report_link_broken\">[log]</span>");
}
@Test
public void testCreateReportFiles_taskWithFilledReport() {
ImmutableMap<String, String> files =
getGeneratedFiles(
EMPTY_PROJECT
.toBuilder()
.addTask(
EMPTY_TASK_SUCCESS
.toBuilder()
.putReport(
"someReport",
FilesWithEntryPoint.create(
ImmutableMap.of(
Paths.get("path", "report.txt"),
toByteArraySupplier("report content")),
Paths.get("path", "report.txt")))
.build())
.build());
assertThat(files).containsEntry("path/report.txt", "report content");
assertThat(files.get("index.html")).contains("<a href=\"path/report.txt\">[someReport]</a>");
}
@Test
public void testCreateReportFiles_taskWithEmptyReport() {
ImmutableMap<String, String> files =
getGeneratedFiles(
EMPTY_PROJECT
.toBuilder()
.addTask(
EMPTY_TASK_SUCCESS
.toBuilder()
.putReport(
"someReport",
FilesWithEntryPoint.create(
ImmutableMap.of(), Paths.get("path", "report.txt")))
.build())
.build());
assertThat(files).doesNotContainKey("path/report.txt");
assertThat(files.get("index.html"))
.contains("<span class=\"report_link_broken\">[someReport]</span>");
}
@Test
public void testCreateReportFiles_taskWithLogAndMultipleReports() {
ImmutableMap<String, String> files =
getGeneratedFiles(
EMPTY_PROJECT
.toBuilder()
.addTask(
EMPTY_TASK_SUCCESS
.toBuilder()
.setUniqueName("my:name")
.setLog(toByteArraySupplier("log data"))
.putReport(
"filledReport",
FilesWithEntryPoint.create(
ImmutableMap.of(
Paths.get("path-filled", "report.txt"),
toByteArraySupplier("report content"),
Paths.get("path-filled", "other", "file.txt"),
toByteArraySupplier("some other content")),
Paths.get("path-filled", "report.txt")))
.putReport(
"emptyReport",
FilesWithEntryPoint.create(
ImmutableMap.of(), Paths.get("path-empty", "report.txt")))
.build())
.build());
assertThat(files).containsEntry("path-filled/report.txt", "report content");
assertThat(files).containsEntry("path-filled/other/file.txt", "some other content");
assertThat(files).containsEntry("logs/my:name.log", "log data");
assertThat(files.get("index.html"))
.contains("<a href=\"path-filled/report.txt\">[filledReport]</a>");
assertThat(files.get("index.html")).contains("<a href=\"logs/my:name.log\">[log]</a>");
assertThat(files.get("index.html"))
.contains("<span class=\"report_link_broken\">[emptyReport]</span>");
}
}

View file

@ -0,0 +1,296 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.gradle.plugin;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.gradle.plugin.GcsPluginUtils.getContentType;
import static google.registry.gradle.plugin.GcsPluginUtils.readFilesWithEntryPoint;
import static google.registry.gradle.plugin.GcsPluginUtils.toByteArraySupplier;
import static google.registry.gradle.plugin.GcsPluginUtils.toNormalizedPath;
import static google.registry.gradle.plugin.GcsPluginUtils.uploadFileToGcs;
import static google.registry.gradle.plugin.GcsPluginUtils.uploadFilesToGcsMultithread;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import com.google.cloud.storage.BlobInfo;
import com.google.cloud.storage.Storage;
import com.google.common.collect.ImmutableMap;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/** Tests for {@link GcsPluginUtilsTest} */
@RunWith(JUnit4.class)
public final class GcsPluginUtilsTest {
@Rule public final TemporaryFolder folder = new TemporaryFolder();
@Test
public void testGetContentType_knownTypes() {
assertThat(getContentType("path/to/file.html")).isEqualTo("text/html");
assertThat(getContentType("path/to/file.htm")).isEqualTo("text/html");
assertThat(getContentType("path/to/file.log")).isEqualTo("text/plain");
assertThat(getContentType("path/to/file.txt")).isEqualTo("text/plain");
assertThat(getContentType("path/to/file.css")).isEqualTo("text/css");
assertThat(getContentType("path/to/file.xml")).isEqualTo("text/xml");
assertThat(getContentType("path/to/file.zip")).isEqualTo("application/zip");
assertThat(getContentType("path/to/file.js")).isEqualTo("text/javascript");
}
@Test
public void testGetContentType_unknownTypes() {
assertThat(getContentType("path/to/file.unknown")).isEqualTo("application/octet-stream");
}
@Test
public void testUploadFileToGcs() {
Storage storage = mock(Storage.class);
uploadFileToGcs(
storage, "my-bucket", Paths.get("my", "filename.txt"), toByteArraySupplier("my data"));
verify(storage)
.create(
BlobInfo.newBuilder("my-bucket", "my/filename.txt")
.setContentType("text/plain")
.build(),
"my data".getBytes(UTF_8));
verifyNoMoreInteractions(storage);
}
@Test
public void testUploadFilesToGcsMultithread() {
Storage storage = mock(Storage.class);
uploadFilesToGcsMultithread(
storage,
"my-bucket",
Paths.get("my", "folder"),
ImmutableMap.of(
Paths.get("some", "index.html"), toByteArraySupplier("some web page"),
Paths.get("some", "style.css"), toByteArraySupplier("some style"),
Paths.get("other", "index.html"), toByteArraySupplier("other web page"),
Paths.get("other", "style.css"), toByteArraySupplier("other style")));
verify(storage)
.create(
BlobInfo.newBuilder("my-bucket", "my/folder/some/index.html")
.setContentType("text/html")
.build(),
"some web page".getBytes(UTF_8));
verify(storage)
.create(
BlobInfo.newBuilder("my-bucket", "my/folder/some/style.css")
.setContentType("text/css")
.build(),
"some style".getBytes(UTF_8));
verify(storage)
.create(
BlobInfo.newBuilder("my-bucket", "my/folder/other/index.html")
.setContentType("text/html")
.build(),
"other web page".getBytes(UTF_8));
verify(storage)
.create(
BlobInfo.newBuilder("my-bucket", "my/folder/other/style.css")
.setContentType("text/css")
.build(),
"other style".getBytes(UTF_8));
verifyNoMoreInteractions(storage);
}
@Test
public void testToByteArraySupplier_string() {
assertThat(toByteArraySupplier("my string").get()).isEqualTo("my string".getBytes(UTF_8));
}
@Test
public void testToByteArraySupplier_stringSupplier() {
assertThat(toByteArraySupplier(() -> "my string").get()).isEqualTo("my string".getBytes(UTF_8));
}
@Test
public void testToByteArraySupplier_file() throws Exception {
folder.newFolder("arbitrary");
File file = folder.newFile("arbitrary/file.txt");
Files.write(file.toPath(), "some data".getBytes(UTF_8));
assertThat(toByteArraySupplier(file).get()).isEqualTo("some data".getBytes(UTF_8));
}
private ImmutableMap<String, String> readAllFiles(FilesWithEntryPoint reportFiles) {
return reportFiles.files().entrySet().stream()
.collect(
toImmutableMap(
entry -> entry.getKey().toString(),
entry -> new String(entry.getValue().get(), UTF_8)));
}
@Test
public void testCreateReportFiles_destinationIsFile() throws Exception {
Path root = toNormalizedPath(folder.newFolder("my", "root"));
folder.newFolder("my", "root", "some", "path");
File destination = folder.newFile("my/root/some/path/file.txt");
Files.write(destination.toPath(), "some data".getBytes(UTF_8));
// Since the entry point is obvious here - any hint given is just ignored.
File ignoredHint = folder.newFile("my/root/ignored.txt");
FilesWithEntryPoint files =
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
assertThat(files.entryPoint().toString()).isEqualTo("some/path/file.txt");
assertThat(readAllFiles(files)).containsExactly("some/path/file.txt", "some data");
}
@Test
public void testCreateReportFiles_destinationDoesntExist() throws Exception {
Path root = toNormalizedPath(folder.newFolder("my", "root"));
File destination = root.resolve("non/existing.txt").toFile();
assertThat(destination.isFile()).isFalse();
assertThat(destination.isDirectory()).isFalse();
// Since there are not files, any hint given is obvioulsy wrong and will be ignored.
File ignoredHint = folder.newFile("my/root/ignored.txt");
FilesWithEntryPoint files =
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
assertThat(files.entryPoint().toString()).isEqualTo("non/existing.txt");
assertThat(files.files()).isEmpty();
}
@Test
public void testCreateReportFiles_noFiles() throws Exception {
Path root = toNormalizedPath(folder.newFolder("my", "root"));
File destination = folder.newFolder("my", "root", "some", "path");
folder.newFolder("my", "root", "some", "path", "a", "b");
folder.newFolder("my", "root", "some", "path", "c");
// Since there are not files, any hint given is obvioulsy wrong and will be ignored.
File ignoredHint = folder.newFile("my/root/ignored.txt");
FilesWithEntryPoint files =
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
assertThat(files.entryPoint().toString()).isEqualTo("some/path");
assertThat(files.files()).isEmpty();
}
@Test
public void testCreateReportFiles_oneFile() throws Exception {
Path root = toNormalizedPath(folder.newFolder("my", "root"));
File destination = folder.newFolder("my", "root", "some", "path");
folder.newFolder("my", "root", "some", "path", "a", "b");
folder.newFolder("my", "root", "some", "path", "c");
Files.write(
folder.newFile("my/root/some/path/a/file.txt").toPath(), "some data".getBytes(UTF_8));
// Since the entry point is obvious here - any hint given is just ignored.
File ignoredHint = folder.newFile("my/root/ignored.txt");
FilesWithEntryPoint files =
readFilesWithEntryPoint(destination, Optional.of(ignoredHint), root);
assertThat(files.entryPoint().toString()).isEqualTo("some/path/a/file.txt");
assertThat(readAllFiles(files)).containsExactly("some/path/a/file.txt", "some data");
}
/**
* Currently tests the "unimplemented" behavior.
*
* <p>TODO(guyben): switch to checking zip file instead.
*/
@Test
public void testCreateReportFiles_multipleFiles_noHint() throws Exception {
Path root = toNormalizedPath(folder.newFolder("my", "root"));
File destination = folder.newFolder("my", "root", "some", "path");
folder.newFolder("my", "root", "some", "path", "a", "b");
folder.newFolder("my", "root", "some", "path", "c");
Files.write(
folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
FilesWithEntryPoint files = readFilesWithEntryPoint(destination, Optional.empty(), root);
assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip");
assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip");
}
/**
* Currently tests the "unimplemented" behavior.
*
* <p>TODO(guyben): switch to checking zip file instead.
*/
@Test
public void testCreateReportFiles_multipleFiles_withBadHint() throws Exception {
Path root = toNormalizedPath(folder.newFolder("my", "root"));
File destination = folder.newFolder("my", "root", "some", "path");
// This entry point points to a directory, which isn't an appropriate entry point
File badEntryPoint = folder.newFolder("my", "root", "some", "path", "a", "b");
folder.newFolder("my", "root", "some", "path", "c");
Files.write(
folder.newFile("my/root/some/path/index.html").toPath(), "some data".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
FilesWithEntryPoint files =
readFilesWithEntryPoint(destination, Optional.of(badEntryPoint), root);
assertThat(files.entryPoint().toString()).isEqualTo("some/path/path.zip");
assertThat(readAllFiles(files).keySet()).containsExactly("some/path/path.zip");
}
@Test
public void testCreateReportFiles_multipleFiles_withGoodHint() throws Exception {
Path root = toNormalizedPath(folder.newFolder("my", "root"));
File destination = folder.newFolder("my", "root", "some", "path");
folder.newFolder("my", "root", "some", "path", "a", "b");
folder.newFolder("my", "root", "some", "path", "c");
// The hint is an actual file nested in the destination directory!
File goodEntryPoint = folder.newFile("my/root/some/path/index.html");
Files.write(goodEntryPoint.toPath(), "some data".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/a/index.html").toPath(), "wrong index".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/c/style.css").toPath(), "css file".getBytes(UTF_8));
Files.write(
folder.newFile("my/root/some/path/my_image.png").toPath(), "images".getBytes(UTF_8));
FilesWithEntryPoint files =
readFilesWithEntryPoint(destination, Optional.of(goodEntryPoint), root);
assertThat(files.entryPoint().toString()).isEqualTo("some/path/index.html");
assertThat(readAllFiles(files))
.containsExactly(
"some/path/index.html", "some data",
"some/path/a/index.html", "wrong index",
"some/path/c/style.css", "css file",
"some/path/my_image.png", "images");
}
}