Support source code cross-referencing for C++ and Java
Use Kythe (https://kythe.io) to build cross reference for the Android
source code. ~generate the input for it during the build. This is done
on demand: if XREF_CORPUS environment variable is set, build emits a
Ninja rule to generate Kythe input for each compilation rule. It
also emits two consolidation rules (`xref_cxx` and `xref_java`),
that depend on all Kythe input generation rules for C++ and Java.
The value of the XREF_CORPUS environment variable is recorded in the
generated files and thus passed to Kythe. For the AOSP master branch it is
`android.googlesource.com/platform/superproject`, so the command to build
all input for Kythe on that branch is:
```
XREF_CORPUS=android.googlesource.com/platform/superproject m xref_cxx xref_java
```
Each Kythe input generation rule generates a single file with .kzip
extension. Individual .kzip files have a lot of common information, so
there will be a post-build consolidation step run to combine them.
The consolidated .kzip file is then passed to Kythe backend.
The tools to generate .kzip files are provided by Kythe (it calls them
'extractors'). We are going to build them in toolbuilding branches
(clang-tools and build-tools) and check them in as binaries into master
and other PDK branches:
For C++, `prebuilts/clang-tools/linux-x86/bin/cxx_extractor`
for Java, `prebuilts/build-tools/common/framework/javac_extractor.jar`
Bug: 121267023
Test: 1) When XREF_CORPUS is set, build generates Ninja rules to create
.kzip files; 2) When XREF_CORPUS is set, building
`xref_cxx`/`xref_java` creates .kzip files; 3) Unless XREF_CORPUS is
set, build generates the same Ninja rules as before
Change-Id: If957b35d7abc82dbfbb3665980e7c34afe7c789e
diff --git a/android/config.go b/android/config.go
index 556ddcc..074dfc7 100644
--- a/android/config.go
+++ b/android/config.go
@@ -756,6 +756,14 @@
return c.IsEnvTrue("RUN_ERROR_PRONE")
}
+func (c *config) XrefCorpusName() string {
+ return c.Getenv("XREF_CORPUS")
+}
+
+func (c *config) EmitXrefRules() bool {
+ return c.XrefCorpusName() != ""
+}
+
// Returns true if -source 1.9 -target 1.9 is being passed to javac
func (c *config) TargetOpenJDK9() bool {
return c.targetOpenJDK9
diff --git a/cc/builder.go b/cc/builder.go
index cbe2c88..2909d51 100644
--- a/cc/builder.go
+++ b/cc/builder.go
@@ -221,6 +221,17 @@
Rspfile: "$out.rsp",
RspfileContent: "$in",
})
+
+ _ = pctx.SourcePathVariable("cxxExtractor",
+ "prebuilts/clang-tools/${config.HostPrebuiltTag}/bin/cxx_extractor")
+ _ = pctx.VariableFunc("kytheCorpus",
+ func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })
+ kytheExtract = pctx.StaticRule("kythe",
+ blueprint.RuleParams{
+ Command: "rm -f $out && KYTHE_CORPUS=${kytheCorpus} KYTHE_OUTPUT_FILE=$out $cxxExtractor $cFlags $in ",
+ CommandDeps: []string{"$cxxExtractor"},
+ },
+ "cFlags")
)
func init() {
@@ -257,6 +268,7 @@
tidy bool
coverage bool
sAbiDump bool
+ emitXrefs bool
systemIncludeFlags string
@@ -281,6 +293,7 @@
tidyFiles android.Paths
coverageFiles android.Paths
sAbiDumpFiles android.Paths
+ kytheFiles android.Paths
}
func (a Objects) Copy() Objects {
@@ -289,6 +302,7 @@
tidyFiles: append(android.Paths{}, a.tidyFiles...),
coverageFiles: append(android.Paths{}, a.coverageFiles...),
sAbiDumpFiles: append(android.Paths{}, a.sAbiDumpFiles...),
+ kytheFiles: append(android.Paths{}, a.kytheFiles...),
}
}
@@ -298,6 +312,7 @@
tidyFiles: append(a.tidyFiles, b.tidyFiles...),
coverageFiles: append(a.coverageFiles, b.coverageFiles...),
sAbiDumpFiles: append(a.sAbiDumpFiles, b.sAbiDumpFiles...),
+ kytheFiles: append(a.kytheFiles, b.kytheFiles...),
}
}
@@ -314,6 +329,10 @@
if flags.coverage {
coverageFiles = make(android.Paths, 0, len(srcFiles))
}
+ var kytheFiles android.Paths
+ if flags.emitXrefs {
+ kytheFiles = make(android.Paths, 0, len(srcFiles))
+ }
commonFlags := strings.Join([]string{
flags.globalFlags,
@@ -401,6 +420,7 @@
coverage := flags.coverage
dump := flags.sAbiDump
rule := cc
+ emitXref := flags.emitXrefs
switch srcFile.Ext() {
case ".s":
@@ -412,6 +432,7 @@
tidy = false
coverage = false
dump = false
+ emitXref = false
case ".c":
ccCmd = "clang"
moduleCflags = cflags
@@ -450,6 +471,22 @@
},
})
+ if emitXref {
+ kytheFile := android.ObjPathWithExt(ctx, subdir, srcFile, "kzip")
+ ctx.Build(pctx, android.BuildParams{
+ Rule: kytheExtract,
+ Description: "Xref C++ extractor " + srcFile.Rel(),
+ Output: kytheFile,
+ Input: srcFile,
+ Implicits: cFlagsDeps,
+ OrderOnly: pathDeps,
+ Args: map[string]string{
+ "cFlags": moduleCflags,
+ },
+ })
+ kytheFiles = append(kytheFiles, kytheFile)
+ }
+
if tidy {
tidyFile := android.ObjPathWithExt(ctx, subdir, srcFile, "tidy")
tidyFiles = append(tidyFiles, tidyFile)
@@ -493,6 +530,7 @@
tidyFiles: tidyFiles,
coverageFiles: coverageFiles,
sAbiDumpFiles: sAbiDumpFiles,
+ kytheFiles: kytheFiles,
}
}
diff --git a/cc/cc.go b/cc/cc.go
index 2cee807..d22363e 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -77,6 +77,7 @@
ctx.TopDown("double_loadable", checkDoubleLoadableLibraries).Parallel()
})
+ android.RegisterSingletonType("kythe_extract_all", kytheExtractAllFactory)
pctx.Import("android/soong/cc/config")
}
@@ -162,6 +163,7 @@
Tidy bool
Coverage bool
SAbiDump bool
+ EmitXrefs bool // If true, generate Ninja rules to generate emitXrefs input files for Kythe
RequiredInstructionSet string
DynamicLinker string
@@ -346,6 +348,10 @@
explicitlyVersioned bool
}
+type xref interface {
+ XrefCcFiles() android.Paths
+}
+
var (
sharedDepTag = dependencyTag{name: "shared", library: true}
sharedExportDepTag = dependencyTag{name: "shared", library: true, reexportFlags: true}
@@ -427,6 +433,8 @@
staticVariant *Module
makeLinkType string
+ // Kythe (source file indexer) paths for this compilation module
+ kytheFiles android.Paths
}
func (c *Module) OutputFile() android.OptionalPath {
@@ -657,6 +665,10 @@
return isBionic(name)
}
+func (c *Module) XrefCcFiles() android.Paths {
+ return c.kytheFiles
+}
+
type baseModuleContext struct {
android.BaseModuleContext
moduleContextImpl
@@ -995,6 +1007,7 @@
flags := Flags{
Toolchain: c.toolchain(ctx),
+ EmitXrefs: ctx.Config().EmitXrefRules(),
}
if c.compiler != nil {
flags = c.compiler.compilerFlags(ctx, flags, deps)
@@ -1060,6 +1073,7 @@
if ctx.Failed() {
return
}
+ c.kytheFiles = objs.kytheFiles
}
if c.linker != nil {
@@ -2366,6 +2380,31 @@
return ctx.Config().PlatformSdkVersion()
}
+func kytheExtractAllFactory() android.Singleton {
+ return &kytheExtractAllSingleton{}
+}
+
+type kytheExtractAllSingleton struct {
+}
+
+func (ks *kytheExtractAllSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+ var xrefTargets android.Paths
+ ctx.VisitAllModules(func(module android.Module) {
+ if ccModule, ok := module.(xref); ok {
+ xrefTargets = append(xrefTargets, ccModule.XrefCcFiles()...)
+ }
+ })
+ // TODO(asmundak): Perhaps emit a rule to output a warning if there were no xrefTargets
+ if len(xrefTargets) > 0 {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: blueprint.Phony,
+ Output: android.PathForPhony(ctx, "xref_cxx"),
+ Inputs: xrefTargets,
+ //Default: true,
+ })
+ }
+}
+
var Bool = proptools.Bool
var BoolDefault = proptools.BoolDefault
var BoolPtr = proptools.BoolPtr
diff --git a/cc/util.go b/cc/util.go
index 2e1bb25..0d1b2f0 100644
--- a/cc/util.go
+++ b/cc/util.go
@@ -74,6 +74,7 @@
coverage: in.Coverage,
tidy: in.Tidy,
sAbiDump: in.SAbiDump,
+ emitXrefs: in.EmitXrefs,
systemIncludeFlags: strings.Join(in.SystemIncludeFlags, " "),
diff --git a/java/builder.go b/java/builder.go
index 22eff7c..f174cf0 100644
--- a/java/builder.go
+++ b/java/builder.go
@@ -62,6 +62,37 @@
"javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",
"outDir", "annoDir", "javaVersion")
+ _ = pctx.VariableFunc("kytheCorpus",
+ func(ctx android.PackageVarContext) string { return ctx.Config().XrefCorpusName() })
+ // Run it with -add-opens=java.base/java.nio=ALL-UNNAMED to avoid JDK9's warning about
+ // "Illegal reflective access by com.google.protobuf.Utf8$UnsafeProcessor ...
+ // to field java.nio.Buffer.address"
+ kytheExtract = pctx.AndroidStaticRule("kythe",
+ blueprint.RuleParams{
+ Command: `${config.ZipSyncCmd} -d $srcJarDir ` +
+ `-l $srcJarDir/list -f "*.java" $srcJars && ` +
+ `( [ ! -s $srcJarDir/list -a ! -s $out.rsp ] || ` +
+ `KYTHE_ROOT_DIRECTORY=. KYTHE_OUTPUT_FILE=$out ` +
+ `KYTHE_CORPUS=${kytheCorpus} ` +
+ `${config.SoongJavacWrapper} ${config.JavaCmd} ` +
+ `--add-opens=java.base/java.nio=ALL-UNNAMED ` +
+ `-jar ${config.JavaKytheExtractorJar} ` +
+ `${config.JavacHeapFlags} ${config.CommonJdkFlags} ` +
+ `$processorpath $processor $javacFlags $bootClasspath $classpath ` +
+ `-source $javaVersion -target $javaVersion ` +
+ `-d $outDir -s $annoDir @$out.rsp @$srcJarDir/list)`,
+ CommandDeps: []string{
+ "${config.JavaCmd}",
+ "${config.JavaKytheExtractorJar}",
+ "${config.ZipSyncCmd}",
+ },
+ CommandOrderOnly: []string{"${config.SoongJavacWrapper}"},
+ Rspfile: "$out.rsp",
+ RspfileContent: "$in",
+ },
+ "javacFlags", "bootClasspath", "classpath", "processorpath", "processor", "srcJars", "srcJarDir",
+ "outDir", "annoDir", "javaVersion")
+
turbine = pctx.AndroidStaticRule("turbine",
blueprint.RuleParams{
Command: `rm -rf "$outDir" && mkdir -p "$outDir" && ` +
@@ -196,6 +227,61 @@
"errorprone", "errorprone")
}
+// Emits the rule to generate Xref input file (.kzip file) for the given set of source files and source jars
+// to compile with given set of builder flags, etc.
+func emitXrefRule(ctx android.ModuleContext, xrefFile android.WritablePath,
+ srcFiles, srcJars android.Paths,
+ flags javaBuilderFlags, deps android.Paths,
+ intermediatesDir string) {
+
+ deps = append(deps, srcJars...)
+
+ var bootClasspath string
+ if flags.javaVersion == "1.9" {
+ var systemModuleDeps android.Paths
+ bootClasspath, systemModuleDeps = flags.systemModules.FormJavaSystemModulesPath(ctx.Device())
+ deps = append(deps, systemModuleDeps...)
+ } else {
+ deps = append(deps, flags.bootClasspath...)
+ if len(flags.bootClasspath) == 0 && ctx.Device() {
+ // explicitly specify -bootclasspath "" if the bootclasspath is empty to
+ // ensure java does not fall back to the default bootclasspath.
+ bootClasspath = `-bootclasspath ""`
+ } else {
+ bootClasspath = flags.bootClasspath.FormJavaClassPath("-bootclasspath")
+ }
+ }
+
+ deps = append(deps, flags.classpath...)
+ deps = append(deps, flags.processorPath...)
+
+ processor := "-proc:none"
+ if flags.processor != "" {
+ processor = "-processor " + flags.processor
+ }
+
+ ctx.Build(pctx,
+ android.BuildParams{
+ Rule: kytheExtract,
+ Description: "Xref Java extractor",
+ Output: xrefFile,
+ Inputs: srcFiles,
+ Implicits: deps,
+ Args: map[string]string{
+ "annoDir": android.PathForModuleOut(ctx, intermediatesDir, "anno").String(),
+ "bootClasspath": bootClasspath,
+ "classpath": flags.classpath.FormJavaClassPath("-classpath"),
+ "javacFlags": flags.javacFlags,
+ "javaVersion": flags.javaVersion,
+ "outDir": android.PathForModuleOut(ctx, "javac", "classes.xref").String(),
+ "processorpath": flags.processorPath.FormJavaClassPath("-processorpath"),
+ "processor": processor,
+ "srcJarDir": android.PathForModuleOut(ctx, intermediatesDir, "srcjars.xref").String(),
+ "srcJars": strings.Join(srcJars.Strings(), " "),
+ },
+ })
+}
+
func TransformJavaToHeaderClasses(ctx android.ModuleContext, outputFile android.WritablePath,
srcFiles, srcJars android.Paths, flags javaBuilderFlags) {
diff --git a/java/config/config.go b/java/config/config.go
index 6a0a10a..cb13744 100644
--- a/java/config/config.go
+++ b/java/config/config.go
@@ -94,6 +94,7 @@
pctx.SourcePathVariable("JlinkCmd", "${JavaToolchain}/jlink")
pctx.SourcePathVariable("JmodCmd", "${JavaToolchain}/jmod")
pctx.SourcePathVariable("JrtFsJar", "${JavaHome}/lib/jrt-fs.jar")
+ pctx.SourcePathVariable("JavaKytheExtractorJar", "prebuilts/build-tools/common/framework/javac_extractor.jar")
pctx.SourcePathVariable("Ziptime", "prebuilts/build-tools/${hostPrebuiltTag}/bin/ziptime")
pctx.SourcePathVariable("GenKotlinBuildFileCmd", "build/soong/scripts/gen-kotlin-build-file.sh")
diff --git a/java/java.go b/java/java.go
index f3e10be..c6f6a34 100644
--- a/java/java.go
+++ b/java/java.go
@@ -50,6 +50,7 @@
android.RegisterModuleType("dex_import", DexImportFactory)
android.RegisterSingletonType("logtags", LogtagsSingleton)
+ android.RegisterSingletonType("kythe_java_extract", kytheExtractJavaFactory)
}
// TODO:
@@ -342,6 +343,9 @@
hiddenAPI
dexpreopter
+
+ // list of the xref extraction files
+ kytheFiles android.Paths
}
func (j *Module) OutputFiles(tag string) (android.Paths, error) {
@@ -382,6 +386,10 @@
CompiledSrcJars() android.Paths
}
+type xref interface {
+ XrefJavaFiles() android.Paths
+}
+
func (j *Module) CompiledSrcs() android.Paths {
return j.compiledJavaSrcs
}
@@ -390,6 +398,10 @@
return j.compiledSrcJars
}
+func (j *Module) XrefJavaFiles() android.Paths {
+ return j.kytheFiles
+}
+
var _ SrcDependency = (*Module)(nil)
func InitJavaModule(module android.DefaultableModule, hod android.HostOrDeviceSupported) {
@@ -1138,6 +1150,12 @@
TransformJavaToClasses(ctx, classes, -1, uniqueSrcFiles, srcJars, flags, extraJarDeps)
jars = append(jars, classes)
}
+ if ctx.Config().EmitXrefRules() {
+ extractionFile := android.PathForModuleOut(ctx, ctx.ModuleName()+".kzip")
+ emitXrefRule(ctx, extractionFile, uniqueSrcFiles, srcJars, flags, extraJarDeps, "xref")
+ j.kytheFiles = append(j.kytheFiles, extractionFile)
+
+ }
if ctx.Failed() {
return
}
@@ -2203,6 +2221,30 @@
return module
}
+func kytheExtractJavaFactory() android.Singleton {
+ return &kytheExtractJavaSingleton{}
+}
+
+type kytheExtractJavaSingleton struct {
+}
+
+func (ks *kytheExtractJavaSingleton) GenerateBuildActions(ctx android.SingletonContext) {
+ var xrefTargets android.Paths
+ ctx.VisitAllModules(func(module android.Module) {
+ if javaModule, ok := module.(xref); ok {
+ xrefTargets = append(xrefTargets, javaModule.XrefJavaFiles()...)
+ }
+ })
+ // TODO(asmundak): perhaps emit a rule to output a warning if there were no xrefTargets
+ if len(xrefTargets) > 0 {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: blueprint.Phony,
+ Output: android.PathForPhony(ctx, "xref_java"),
+ Inputs: xrefTargets,
+ })
+ }
+}
+
var Bool = proptools.Bool
var BoolDefault = proptools.BoolDefault
var String = proptools.String