Add publish functionality to billing pipeline

This closes the end-to-end billing pipeline, allowing us to share generated detail reports with registrars via Drive and e-mail the invoicing team a link to the generated invoice.

This also factors out the email configs from ICANN reporting into the common 'misc' config, since we'll likely need alert e-mails for future periodic tasks.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=180805972
This commit is contained in:
larryruili 2018-01-04 09:16:51 -08:00 committed by jianglai
parent 27f12b9390
commit ab5e16ab67
25 changed files with 721 additions and 95 deletions

View file

@ -21,7 +21,10 @@ import com.google.appengine.tools.cloudstorage.GcsFileOptions;
import com.google.appengine.tools.cloudstorage.GcsFileOptions.Builder;
import com.google.appengine.tools.cloudstorage.GcsFilename;
import com.google.appengine.tools.cloudstorage.GcsService;
import com.google.appengine.tools.cloudstorage.ListOptions;
import com.google.appengine.tools.cloudstorage.ListResult;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
@ -72,6 +75,30 @@ public class GcsUtils {
gcsService.createOrReplace(filename, getOptions(filename), ByteBuffer.wrap(bytes));
}
/**
* Returns a list of all object names within a bucket for a given prefix.
*
* <p>Note this also strips the provided prefix from the object, leaving only the object name
* (i.e. if the bucket hello contains gs://hello/world/myobj.txt and gs://hello/world/a/b.csv,
* then listFolderObjects("hello", "world/") would return {"myobj.txt", "a/b.csv"}.
*
* @param bucketName the name of the bucket, with no gs:// namespace or trailing slashes
* @param prefix the string prefix all objects in the bucket should start with
*/
public ImmutableList<String> listFolderObjects(String bucketName, String prefix)
throws IOException {
ListResult result =
gcsService.list(bucketName, new ListOptions.Builder().setPrefix(prefix).build());
final ImmutableList.Builder<String> builder = new ImmutableList.Builder<>();
result.forEachRemaining(
listItem -> {
if (!listItem.isDirectory()) {
builder.add(listItem.getName().replaceFirst(prefix, ""));
}
});
return builder.build();
}
/** Returns {@code true} if a file exists and is non-empty on Google Cloud Storage. */
public boolean existsAndNotEmpty(GcsFilename file) {
GcsFileMetadata metadata;