Create kzip file for Kythe cross-referencing (#424)

This is set up per b/141716384.

TESTED=Tested on alpha and successfully uploaded the merged kzip file to
GCS.
This commit is contained in:
Lai Jiang 2020-01-03 16:57:39 -05:00 committed by GitHub
parent 566b3f38ba
commit 0a6d2a9f47
8 changed files with 153 additions and 56 deletions

View file

@ -179,6 +179,16 @@ allprojects {
mavenCentral()
}
}
if (rootProject.enableCrossReferencing.toBoolean()) {
gradle.projectsEvaluated {
tasks.withType(JavaCompile) {
options.fork = true
options.forkOptions.executable =
"${project.rootDir}/kythe/extractors/javac-wrapper.sh"
}
}
}
}
task runPresubmits(type: Exec) {

View file

@ -10,11 +10,11 @@ com.fasterxml.jackson.core:jackson-annotations:2.9.10
com.fasterxml.jackson.core:jackson-core:2.9.10
com.fasterxml.jackson.core:jackson-databind:2.9.10
com.fasterxml:classmate:1.3.4
com.google.api-client:google-api-client-appengine:1.30.6
com.google.api-client:google-api-client-appengine:1.29.0
com.google.api-client:google-api-client-jackson2:1.27.0
com.google.api-client:google-api-client-java6:1.27.0
com.google.api-client:google-api-client-servlet:1.30.6
com.google.api-client:google-api-client:1.30.6
com.google.api-client:google-api-client-servlet:1.29.0
com.google.api-client:google-api-client:1.29.2
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.44.0
com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:0.38.0
com.google.api.grpc:grpc-google-cloud-bigtable-v2:0.38.0
@ -84,22 +84,22 @@ com.google.guava:failureaccess:1.0.1
com.google.guava:guava:28.1-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.google.gwt:gwt-user:2.8.2
com.google.http-client:google-http-client-appengine:1.32.1
com.google.http-client:google-http-client-jackson2:1.32.1
com.google.http-client:google-http-client-appengine:1.29.2
com.google.http-client:google-http-client-jackson2:1.30.1
com.google.http-client:google-http-client-jackson:1.20.0
com.google.http-client:google-http-client-protobuf:1.20.0
com.google.http-client:google-http-client:1.32.1
com.google.http-client:google-http-client:1.30.1
com.google.inject.extensions:guice-multibindings:4.1.0
com.google.inject:guice:4.1.0
com.google.j2objc:j2objc-annotations:1.3
com.google.jsinterop:jsinterop-annotations:1.0.2
com.google.monitoring-client:metrics:1.0.6
com.google.monitoring-client:stackdriver:1.0.6
com.google.oauth-client:google-oauth-client-appengine:1.30.4
com.google.oauth-client:google-oauth-client-appengine:1.29.0
com.google.oauth-client:google-oauth-client-java6:1.28.0
com.google.oauth-client:google-oauth-client-jetty:1.28.0
com.google.oauth-client:google-oauth-client-servlet:1.30.4
com.google.oauth-client:google-oauth-client:1.30.4
com.google.oauth-client:google-oauth-client-servlet:1.29.0
com.google.oauth-client:google-oauth-client:1.29.2
com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-5
com.google.protobuf:protobuf-java-util:3.6.1
com.google.protobuf:protobuf-java:3.6.1
@ -124,7 +124,7 @@ io.dropwizard.metrics:metrics-core:3.1.2
io.grpc:grpc-all:1.17.1
io.grpc:grpc-alts:1.17.1
io.grpc:grpc-auth:1.17.1
io.grpc:grpc-context:1.22.1
io.grpc:grpc-context:1.19.0
io.grpc:grpc-core:1.17.1
io.grpc:grpc-grpclb:1.17.1
io.grpc:grpc-netty-shaded:1.17.1
@ -146,10 +146,10 @@ io.netty:netty-handler:4.1.30.Final
io.netty:netty-resolver:4.1.30.Final
io.netty:netty-tcnative-boringssl-static:2.0.17.Final
io.netty:netty-transport:4.1.30.Final
io.opencensus:opencensus-api:0.24.0
io.opencensus:opencensus-api:0.21.0
io.opencensus:opencensus-contrib-grpc-metrics:0.17.0
io.opencensus:opencensus-contrib-grpc-util:0.17.0
io.opencensus:opencensus-contrib-http-util:0.24.0
io.opencensus:opencensus-contrib-http-util:0.21.0
it.unimi.dsi:fastutil:6.5.16
javax.activation:activation:1.1
javax.activation:javax.activation-api:1.2.0
@ -183,8 +183,8 @@ org.apache.beam:beam-vendor-grpc-1_21_0:0.1
org.apache.beam:beam-vendor-guava-26_0-jre:0.1
org.apache.commons:commons-compress:1.19
org.apache.commons:commons-lang3:3.5
org.apache.httpcomponents:httpclient:4.5.10
org.apache.httpcomponents:httpcore:4.4.12
org.apache.httpcomponents:httpclient:4.5.8
org.apache.httpcomponents:httpcore:4.4.11
org.bouncycastle:bcpg-jdk15on:1.61
org.bouncycastle:bcprov-jdk15on:1.61
org.checkerframework:checker-qual:2.8.1

View file

@ -10,11 +10,11 @@ com.fasterxml.jackson.core:jackson-annotations:2.9.10
com.fasterxml.jackson.core:jackson-core:2.9.10
com.fasterxml.jackson.core:jackson-databind:2.9.10
com.fasterxml:classmate:1.3.4
com.google.api-client:google-api-client-appengine:1.30.6
com.google.api-client:google-api-client-appengine:1.29.0
com.google.api-client:google-api-client-jackson2:1.27.0
com.google.api-client:google-api-client-java6:1.27.0
com.google.api-client:google-api-client-servlet:1.30.6
com.google.api-client:google-api-client:1.30.6
com.google.api-client:google-api-client-servlet:1.29.0
com.google.api-client:google-api-client:1.29.2
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.44.0
com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:0.38.0
com.google.api.grpc:grpc-google-cloud-bigtable-v2:0.38.0
@ -83,22 +83,22 @@ com.google.guava:failureaccess:1.0.1
com.google.guava:guava:28.1-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.google.gwt:gwt-user:2.8.2
com.google.http-client:google-http-client-appengine:1.32.1
com.google.http-client:google-http-client-jackson2:1.32.1
com.google.http-client:google-http-client-appengine:1.29.2
com.google.http-client:google-http-client-jackson2:1.30.1
com.google.http-client:google-http-client-jackson:1.20.0
com.google.http-client:google-http-client-protobuf:1.20.0
com.google.http-client:google-http-client:1.32.1
com.google.http-client:google-http-client:1.30.1
com.google.inject.extensions:guice-multibindings:4.1.0
com.google.inject:guice:4.1.0
com.google.j2objc:j2objc-annotations:1.3
com.google.jsinterop:jsinterop-annotations:1.0.2
com.google.monitoring-client:metrics:1.0.6
com.google.monitoring-client:stackdriver:1.0.6
com.google.oauth-client:google-oauth-client-appengine:1.30.4
com.google.oauth-client:google-oauth-client-appengine:1.29.0
com.google.oauth-client:google-oauth-client-java6:1.28.0
com.google.oauth-client:google-oauth-client-jetty:1.28.0
com.google.oauth-client:google-oauth-client-servlet:1.30.4
com.google.oauth-client:google-oauth-client:1.30.4
com.google.oauth-client:google-oauth-client-servlet:1.29.0
com.google.oauth-client:google-oauth-client:1.29.2
com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-5
com.google.protobuf:protobuf-java-util:3.6.1
com.google.protobuf:protobuf-java:3.6.1
@ -123,7 +123,7 @@ io.dropwizard.metrics:metrics-core:3.1.2
io.grpc:grpc-all:1.17.1
io.grpc:grpc-alts:1.17.1
io.grpc:grpc-auth:1.17.1
io.grpc:grpc-context:1.22.1
io.grpc:grpc-context:1.19.0
io.grpc:grpc-core:1.17.1
io.grpc:grpc-netty-shaded:1.17.1
io.grpc:grpc-netty:1.17.1
@ -144,10 +144,10 @@ io.netty:netty-handler:4.1.30.Final
io.netty:netty-resolver:4.1.30.Final
io.netty:netty-tcnative-boringssl-static:2.0.17.Final
io.netty:netty-transport:4.1.30.Final
io.opencensus:opencensus-api:0.24.0
io.opencensus:opencensus-api:0.21.0
io.opencensus:opencensus-contrib-grpc-metrics:0.17.0
io.opencensus:opencensus-contrib-grpc-util:0.17.0
io.opencensus:opencensus-contrib-http-util:0.24.0
io.opencensus:opencensus-contrib-http-util:0.21.0
it.unimi.dsi:fastutil:6.5.16
javax.activation:activation:1.1
javax.activation:javax.activation-api:1.2.0
@ -181,8 +181,8 @@ org.apache.beam:beam-vendor-grpc-1_21_0:0.1
org.apache.beam:beam-vendor-guava-26_0-jre:0.1
org.apache.commons:commons-compress:1.19
org.apache.commons:commons-lang3:3.5
org.apache.httpcomponents:httpclient:4.5.10
org.apache.httpcomponents:httpcore:4.4.12
org.apache.httpcomponents:httpclient:4.5.8
org.apache.httpcomponents:httpcore:4.4.11
org.bouncycastle:bcpg-jdk15on:1.61
org.bouncycastle:bcprov-jdk15on:1.61
org.checkerframework:checker-qual:2.8.1

View file

@ -10,11 +10,11 @@ com.fasterxml.jackson.core:jackson-annotations:2.9.10
com.fasterxml.jackson.core:jackson-core:2.9.10
com.fasterxml.jackson.core:jackson-databind:2.9.10
com.fasterxml:classmate:1.3.4
com.google.api-client:google-api-client-appengine:1.30.6
com.google.api-client:google-api-client-appengine:1.29.0
com.google.api-client:google-api-client-jackson2:1.27.0
com.google.api-client:google-api-client-java6:1.27.0
com.google.api-client:google-api-client-servlet:1.30.6
com.google.api-client:google-api-client:1.30.6
com.google.api-client:google-api-client-servlet:1.29.0
com.google.api-client:google-api-client:1.29.2
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.44.0
com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:0.38.0
com.google.api.grpc:grpc-google-cloud-bigtable-v2:0.38.0
@ -84,22 +84,22 @@ com.google.guava:failureaccess:1.0.1
com.google.guava:guava:28.1-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.google.gwt:gwt-user:2.8.2
com.google.http-client:google-http-client-appengine:1.32.1
com.google.http-client:google-http-client-jackson2:1.32.1
com.google.http-client:google-http-client-appengine:1.29.2
com.google.http-client:google-http-client-jackson2:1.30.1
com.google.http-client:google-http-client-jackson:1.20.0
com.google.http-client:google-http-client-protobuf:1.20.0
com.google.http-client:google-http-client:1.32.1
com.google.http-client:google-http-client:1.30.1
com.google.inject.extensions:guice-multibindings:4.1.0
com.google.inject:guice:4.1.0
com.google.j2objc:j2objc-annotations:1.3
com.google.jsinterop:jsinterop-annotations:1.0.2
com.google.monitoring-client:metrics:1.0.6
com.google.monitoring-client:stackdriver:1.0.6
com.google.oauth-client:google-oauth-client-appengine:1.30.4
com.google.oauth-client:google-oauth-client-appengine:1.29.0
com.google.oauth-client:google-oauth-client-java6:1.28.0
com.google.oauth-client:google-oauth-client-jetty:1.28.0
com.google.oauth-client:google-oauth-client-servlet:1.30.4
com.google.oauth-client:google-oauth-client:1.30.4
com.google.oauth-client:google-oauth-client-servlet:1.29.0
com.google.oauth-client:google-oauth-client:1.29.2
com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-5
com.google.protobuf:protobuf-java-util:3.6.1
com.google.protobuf:protobuf-java:3.6.1
@ -124,7 +124,7 @@ io.dropwizard.metrics:metrics-core:3.1.2
io.grpc:grpc-all:1.17.1
io.grpc:grpc-alts:1.17.1
io.grpc:grpc-auth:1.17.1
io.grpc:grpc-context:1.22.1
io.grpc:grpc-context:1.19.0
io.grpc:grpc-core:1.17.1
io.grpc:grpc-grpclb:1.17.1
io.grpc:grpc-netty-shaded:1.17.1
@ -146,10 +146,10 @@ io.netty:netty-handler:4.1.30.Final
io.netty:netty-resolver:4.1.30.Final
io.netty:netty-tcnative-boringssl-static:2.0.17.Final
io.netty:netty-transport:4.1.30.Final
io.opencensus:opencensus-api:0.24.0
io.opencensus:opencensus-api:0.21.0
io.opencensus:opencensus-contrib-grpc-metrics:0.17.0
io.opencensus:opencensus-contrib-grpc-util:0.17.0
io.opencensus:opencensus-contrib-http-util:0.24.0
io.opencensus:opencensus-contrib-http-util:0.21.0
it.unimi.dsi:fastutil:6.5.16
javax.activation:activation:1.1
javax.activation:javax.activation-api:1.2.0
@ -183,8 +183,8 @@ org.apache.beam:beam-vendor-grpc-1_21_0:0.1
org.apache.beam:beam-vendor-guava-26_0-jre:0.1
org.apache.commons:commons-compress:1.19
org.apache.commons:commons-lang3:3.5
org.apache.httpcomponents:httpclient:4.5.10
org.apache.httpcomponents:httpcore:4.4.12
org.apache.httpcomponents:httpclient:4.5.8
org.apache.httpcomponents:httpcore:4.4.11
org.bouncycastle:bcpg-jdk15on:1.61
org.bouncycastle:bcprov-jdk15on:1.61
org.checkerframework:checker-qual:2.8.1

View file

@ -10,11 +10,11 @@ com.fasterxml.jackson.core:jackson-annotations:2.9.10
com.fasterxml.jackson.core:jackson-core:2.9.10
com.fasterxml.jackson.core:jackson-databind:2.9.10
com.fasterxml:classmate:1.3.4
com.google.api-client:google-api-client-appengine:1.30.6
com.google.api-client:google-api-client-appengine:1.29.0
com.google.api-client:google-api-client-jackson2:1.27.0
com.google.api-client:google-api-client-java6:1.27.0
com.google.api-client:google-api-client-servlet:1.30.6
com.google.api-client:google-api-client:1.30.6
com.google.api-client:google-api-client-servlet:1.29.0
com.google.api-client:google-api-client:1.29.2
com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.44.0
com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2:0.38.0
com.google.api.grpc:grpc-google-cloud-bigtable-v2:0.38.0
@ -84,22 +84,22 @@ com.google.guava:failureaccess:1.0.1
com.google.guava:guava:28.1-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.google.gwt:gwt-user:2.8.2
com.google.http-client:google-http-client-appengine:1.32.1
com.google.http-client:google-http-client-jackson2:1.32.1
com.google.http-client:google-http-client-appengine:1.29.2
com.google.http-client:google-http-client-jackson2:1.30.1
com.google.http-client:google-http-client-jackson:1.20.0
com.google.http-client:google-http-client-protobuf:1.20.0
com.google.http-client:google-http-client:1.32.1
com.google.http-client:google-http-client:1.30.1
com.google.inject.extensions:guice-multibindings:4.1.0
com.google.inject:guice:4.1.0
com.google.j2objc:j2objc-annotations:1.3
com.google.jsinterop:jsinterop-annotations:1.0.2
com.google.monitoring-client:metrics:1.0.6
com.google.monitoring-client:stackdriver:1.0.6
com.google.oauth-client:google-oauth-client-appengine:1.30.4
com.google.oauth-client:google-oauth-client-appengine:1.29.0
com.google.oauth-client:google-oauth-client-java6:1.28.0
com.google.oauth-client:google-oauth-client-jetty:1.28.0
com.google.oauth-client:google-oauth-client-servlet:1.30.4
com.google.oauth-client:google-oauth-client:1.30.4
com.google.oauth-client:google-oauth-client-servlet:1.29.0
com.google.oauth-client:google-oauth-client:1.29.2
com.google.protobuf.nano:protobuf-javanano:3.0.0-alpha-5
com.google.protobuf:protobuf-java-util:3.6.1
com.google.protobuf:protobuf-java:3.6.1
@ -124,7 +124,7 @@ io.dropwizard.metrics:metrics-core:3.1.2
io.grpc:grpc-all:1.17.1
io.grpc:grpc-alts:1.17.1
io.grpc:grpc-auth:1.17.1
io.grpc:grpc-context:1.22.1
io.grpc:grpc-context:1.19.0
io.grpc:grpc-core:1.17.1
io.grpc:grpc-grpclb:1.17.1
io.grpc:grpc-netty-shaded:1.17.1
@ -146,10 +146,10 @@ io.netty:netty-handler:4.1.30.Final
io.netty:netty-resolver:4.1.30.Final
io.netty:netty-tcnative-boringssl-static:2.0.17.Final
io.netty:netty-transport:4.1.30.Final
io.opencensus:opencensus-api:0.24.0
io.opencensus:opencensus-api:0.21.0
io.opencensus:opencensus-contrib-grpc-metrics:0.17.0
io.opencensus:opencensus-contrib-grpc-util:0.17.0
io.opencensus:opencensus-contrib-http-util:0.24.0
io.opencensus:opencensus-contrib-http-util:0.21.0
it.unimi.dsi:fastutil:6.5.16
javax.activation:activation:1.1
javax.activation:javax.activation-api:1.2.0
@ -183,8 +183,8 @@ org.apache.beam:beam-vendor-grpc-1_21_0:0.1
org.apache.beam:beam-vendor-guava-26_0-jre:0.1
org.apache.commons:commons-compress:1.19
org.apache.commons:commons-lang3:3.5
org.apache.httpcomponents:httpclient:4.5.10
org.apache.httpcomponents:httpcore:4.4.12
org.apache.httpcomponents:httpclient:4.5.8
org.apache.httpcomponents:httpcore:4.4.11
org.bouncycastle:bcpg-jdk15on:1.61
org.bouncycastle:bcprov-jdk15on:1.61
org.checkerframework:checker-qual:2.8.1

View file

@ -7,6 +7,7 @@ uploaderMultithreadedUpload=
verboseTestOutput=false
flowDocsFile=
enableDependencyLocking=true
enableCrossReferencing=false
# Comma separated list of test patterns, if specified run only these.
testFilter=

View file

@ -0,0 +1,61 @@
# To run the build locally, install cloud-build-local first.
# See: https://cloud.google.com/cloud-build/docs/build-debug-locally
# In the root of a nomulus source tree, run:
# cloud-build-local --config=cloudbuild-kythe.yaml --dryrun=false \
# --substitutions _KYTHE_VERSION=[kythe_version],COMMIT_HASH=[hash] .
# This will download kythe version ${kythe_version} (must be higher than
# v0.0.39 and build a ${hash}.kzip file for Kythe to enable cross referencing.
#
# To manually trigger a build on GCB, run:
# gcloud builds submit --config cloudbuild-kythe.yaml --substitutions \
# _KYTHE_VERSION[kythe_version] .
#
# To trigger a build automatically, follow the instructions below and add a trigger:
# https://cloud.google.com/cloud-build/docs/running-builds/automate-builds
steps:
# Download Kythe
- name: 'openjdk:11'
entrypoint: /bin/bash
args:
- -c
- |
wget -q \
https://github.com/kythe/kythe/releases/download/${_KYTHE_VERSION}/kythe-${_KYTHE_VERSION}.tar.gz
tar xvf kythe-${_KYTHE_VERSION}.tar.gz
rm kythe-${_KYTHE_VERSION}.tar.gz
mv kythe-${_KYTHE_VERSION} kythe
# Build Nomulus with the Kythe wrapper
- name: 'openjdk:11'
entrypoint: /bin/bash
args:
- -c
- |
export REAL_JAVAC=$(which javac)
export JAVAC_EXTRACTOR_JAR="$${PWD}/kythe/extractors/javac_extractor.jar"
export KYTHE_VNAMES="$${PWD}/vnames.json"
export KYTHE_ROOT_DIRECTORY="$${PWD}"
export KYTHE_OUTPUT_DIRECTORY="$${PWD}/kythe_output"
mkdir -p $${KYTHE_OUTPUT_DIRECTORY}
mkdir -p $${KYTHE_OUTPUT_DIRECTORY}/merged
./gradlew clean testClasses -x compileProdJS \
-Dno_werror=true -PenableCrossReferencing=true
# Merge kzip files
- name: 'openjdk:11'
entrypoint: /bin/bash
args:
- -c
- |
export KYTHE_OUTPUT_DIRECTORY="$${PWD}/kythe_output"
./kythe/tools/kzip merge \
--output $${KYTHE_OUTPUT_DIRECTORY}/merged/${COMMIT_SHA}.kzip \
$${KYTHE_OUTPUT_DIRECTORY}/*.kzip
artifacts:
objects:
location: 'gs://${PROJECT_ID}-codesearch'
paths:
- 'kythe_output/merged/${COMMIT_SHA}.kzip'
timeout: 3600s
options:
machineType: 'N1_HIGHCPU_8'

25
vnames.json Normal file
View file

@ -0,0 +1,25 @@
[
{
"pattern": "(build/[^/]+)/(.*)",
"vname": {
"corpus": "github.com/google/nomulus",
"path": "@2@",
"root": "@1@"
}
},
{
"pattern": ".*/.gradle/caches/(.*)",
"vname": {
"corpus": "github.com/google/nomulus",
"path": "@1@",
"root": ".gradle/caches"
}
},
{
"pattern": "(.*)",
"vname": {
"corpus": "github.com/google/nomulus",
"path": "@1@"
}
}
]