Update ide_query script to new format
This includes using a separate (but backward compatible) proto for the cc_analyzer since some of the fields/messages were removed in the new ide_query format.
Tested the ide_query.go with the old and the new cc_analyzer to ensure backward compatibility.
Change-Id: If149f5f9dd88a8f50c184274e8b258dfce117498
diff --git a/tools/ide_query/ide_query.go b/tools/ide_query/ide_query.go
index de84fbe..23c7abd 100644
--- a/tools/ide_query/ide_query.go
+++ b/tools/ide_query/ide_query.go
@@ -33,6 +33,7 @@
"strings"
"google.golang.org/protobuf/proto"
+ apb "ide_query/cc_analyzer_proto"
pb "ide_query/ide_query_proto"
)
@@ -42,9 +43,6 @@
RepoDir string
OutDir string
ClangToolsRoot string
-
- CcFiles []string
- JavaFiles []string
}
// LunchTarget is a parsed Android lunch target.
@@ -83,7 +81,7 @@
func main() {
var env Env
- env.OutDir = os.Getenv("OUT_DIR")
+ env.OutDir = strings.TrimSuffix(os.Getenv("OUT_DIR"), "/")
env.RepoDir = os.Getenv("ANDROID_BUILD_TOP")
env.ClangToolsRoot = os.Getenv("PREBUILTS_CLANG_TOOLS_ROOT")
flag.Var(&env.LunchTarget, "lunch_target", "The lunch target to query")
@@ -95,12 +93,13 @@
return
}
+ var ccFiles, javaFiles []string
for _, f := range files {
switch {
case strings.HasSuffix(f, ".java") || strings.HasSuffix(f, ".kt"):
- env.JavaFiles = append(env.JavaFiles, f)
+ javaFiles = append(javaFiles, f)
case strings.HasSuffix(f, ".cc") || strings.HasSuffix(f, ".cpp") || strings.HasSuffix(f, ".h"):
- env.CcFiles = append(env.CcFiles, f)
+ ccFiles = append(ccFiles, f)
default:
log.Printf("File %q is supported - will be skipped.", f)
}
@@ -110,28 +109,54 @@
// TODO(michaelmerg): Figure out if module_bp_java_deps.json and compile_commands.json is outdated.
runMake(ctx, env, "nothing")
- javaModules, javaFileToModuleMap, err := loadJavaModules(&env)
+ javaModules, err := loadJavaModules(env)
if err != nil {
log.Printf("Failed to load java modules: %v", err)
}
- toMake := getJavaTargets(javaFileToModuleMap)
- ccTargets, status := getCCTargets(ctx, &env)
- if status != nil && status.Code != pb.Status_OK {
- log.Fatalf("Failed to query cc targets: %v", *status.Message)
- }
- toMake = append(toMake, ccTargets...)
- fmt.Fprintf(os.Stderr, "Running make for modules: %v\n", strings.Join(toMake, ", "))
- if err := runMake(ctx, env, toMake...); err != nil {
- log.Printf("Building deps failed: %v", err)
+ var targets []string
+ javaTargetsByFile := findJavaModules(javaFiles, javaModules)
+ for _, t := range javaTargetsByFile {
+ targets = append(targets, t)
}
- res := getJavaInputs(&env, javaModules, javaFileToModuleMap)
- ccAnalysis := getCCInputs(ctx, &env)
- proto.Merge(res, ccAnalysis)
+ ccTargets, err := getCCTargets(ctx, env, ccFiles)
+ if err != nil {
+ log.Fatalf("Failed to query cc targets: %v", err)
+ }
+ targets = append(targets, ccTargets...)
+ if len(targets) == 0 {
+ fmt.Println("No targets found.")
+ os.Exit(1)
+ return
+ }
- res.BuildArtifactRoot = env.OutDir
- data, err := proto.Marshal(res)
+ fmt.Fprintf(os.Stderr, "Running make for modules: %v\n", strings.Join(targets, ", "))
+ if err := runMake(ctx, env, targets...); err != nil {
+ log.Printf("Building modules failed: %v", err)
+ }
+
+ var analysis pb.IdeAnalysis
+ results, units := getJavaInputs(env, javaTargetsByFile, javaModules)
+ analysis.Results = results
+ analysis.Units = units
+ if err != nil && analysis.Error == nil {
+ analysis.Error = &pb.AnalysisError{
+ ErrorMessage: err.Error(),
+ }
+ }
+
+ results, units, err = getCCInputs(ctx, env, ccFiles)
+ analysis.Results = append(analysis.Results, results...)
+ analysis.Units = append(analysis.Units, units...)
+ if err != nil && analysis.Error == nil {
+ analysis.Error = &pb.AnalysisError{
+ ErrorMessage: err.Error(),
+ }
+ }
+
+ analysis.BuildOutDir = env.OutDir
+ data, err := proto.Marshal(&analysis)
if err != nil {
log.Fatalf("Failed to marshal result proto: %v", err)
}
@@ -141,22 +166,22 @@
log.Fatalf("Failed to write result proto: %v", err)
}
- for _, s := range res.Sources {
- fmt.Fprintf(os.Stderr, "%s: %v (Deps: %d, Generated: %d)\n", s.GetPath(), s.GetStatus(), len(s.GetDeps()), len(s.GetGenerated()))
+ for _, r := range analysis.Results {
+ fmt.Fprintf(os.Stderr, "%s: %+v\n", r.GetSourceFilePath(), r.GetStatus())
}
}
-func repoState(env *Env) *pb.RepoState {
+func repoState(env Env, filePaths []string) *apb.RepoState {
const compDbPath = "soong/development/ide/compdb/compile_commands.json"
- return &pb.RepoState{
+ return &apb.RepoState{
RepoDir: env.RepoDir,
- ActiveFilePath: env.CcFiles,
+ ActiveFilePath: filePaths,
OutDir: env.OutDir,
CompDbPath: path.Join(env.OutDir, compDbPath),
}
}
-func runCCanalyzer(ctx context.Context, env *Env, mode string, in []byte) ([]byte, error) {
+func runCCanalyzer(ctx context.Context, env Env, mode string, in []byte) ([]byte, error) {
ccAnalyzerPath := path.Join(env.ClangToolsRoot, "bin/ide_query_cc_analyzer")
outBuffer := new(bytes.Buffer)
@@ -176,127 +201,205 @@
}
// Execute cc_analyzer and get all the targets that needs to be build for analyzing files.
-func getCCTargets(ctx context.Context, env *Env) ([]string, *pb.Status) {
- state := repoState(env)
- bytes, err := proto.Marshal(state)
+func getCCTargets(ctx context.Context, env Env, filePaths []string) ([]string, error) {
+ state, err := proto.Marshal(repoState(env, filePaths))
if err != nil {
log.Fatalln("Failed to serialize state:", err)
}
- resp := new(pb.DepsResponse)
- result, err := runCCanalyzer(ctx, env, "deps", bytes)
- if marshal_err := proto.Unmarshal(result, resp); marshal_err != nil {
- return nil, &pb.Status{
- Code: pb.Status_FAILURE,
- Message: proto.String("Malformed response from cc_analyzer: " + marshal_err.Error()),
- }
+ resp := new(apb.DepsResponse)
+ result, err := runCCanalyzer(ctx, env, "deps", state)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := proto.Unmarshal(result, resp); err != nil {
+ return nil, fmt.Errorf("malformed response from cc_analyzer: %v", err)
}
var targets []string
- if resp.Status != nil && resp.Status.Code != pb.Status_OK {
- return targets, resp.Status
+ if resp.Status != nil && resp.Status.Code != apb.Status_OK {
+ return targets, fmt.Errorf("cc_analyzer failed: %v", resp.Status.Message)
}
+
for _, deps := range resp.Deps {
targets = append(targets, deps.BuildTarget...)
}
-
- status := &pb.Status{Code: pb.Status_OK}
- if err != nil {
- status = &pb.Status{
- Code: pb.Status_FAILURE,
- Message: proto.String(err.Error()),
- }
- }
- return targets, status
+ return targets, nil
}
-func getCCInputs(ctx context.Context, env *Env) *pb.IdeAnalysis {
- state := repoState(env)
- bytes, err := proto.Marshal(state)
+func getCCInputs(ctx context.Context, env Env, filePaths []string) ([]*pb.AnalysisResult, []*pb.BuildableUnit, error) {
+ state, err := proto.Marshal(repoState(env, filePaths))
if err != nil {
log.Fatalln("Failed to serialize state:", err)
}
- resp := new(pb.IdeAnalysis)
- result, err := runCCanalyzer(ctx, env, "inputs", bytes)
- if marshal_err := proto.Unmarshal(result, resp); marshal_err != nil {
- resp.Status = &pb.Status{
- Code: pb.Status_FAILURE,
- Message: proto.String("Malformed response from cc_analyzer: " + marshal_err.Error()),
- }
- return resp
+ resp := new(apb.IdeAnalysis)
+ result, err := runCCanalyzer(ctx, env, "inputs", state)
+ if err != nil {
+ return nil, nil, fmt.Errorf("cc_analyzer failed:", err)
+ }
+ if err := proto.Unmarshal(result, resp); err != nil {
+ return nil, nil, fmt.Errorf("malformed response from cc_analyzer: %v", err)
+ }
+ if resp.Status != nil && resp.Status.Code != apb.Status_OK {
+ return nil, nil, fmt.Errorf("cc_analyzer failed: %v", resp.Status.Message)
}
- if err != nil && (resp.Status == nil || resp.Status.Code == pb.Status_OK) {
- resp.Status = &pb.Status{
- Code: pb.Status_FAILURE,
- Message: proto.String(err.Error()),
+ var results []*pb.AnalysisResult
+ var units []*pb.BuildableUnit
+ for _, s := range resp.Sources {
+ status := &pb.AnalysisResult_Status{
+ Code: pb.AnalysisResult_Status_CODE_OK,
}
- }
- return resp
-}
-
-func getJavaTargets(javaFileToModuleMap map[string]*javaModule) []string {
- var targets []string
- for _, m := range javaFileToModuleMap {
- targets = append(targets, m.Name)
- }
- return targets
-}
-
-func getJavaInputs(env *Env, javaModules map[string]*javaModule, javaFileToModuleMap map[string]*javaModule) *pb.IdeAnalysis {
- var sources []*pb.SourceFile
- type depsAndGenerated struct {
- Deps []string
- Generated []*pb.GeneratedFile
- }
- moduleToDeps := make(map[string]*depsAndGenerated)
- for _, f := range env.JavaFiles {
- file := &pb.SourceFile{
- Path: f,
- }
- sources = append(sources, file)
-
- m := javaFileToModuleMap[f]
- if m == nil {
- file.Status = &pb.Status{
- Code: pb.Status_FAILURE,
- Message: proto.String("File not found in any module."),
- }
- continue
+ if s.GetStatus().GetCode() != apb.Status_OK {
+ status.Code = pb.AnalysisResult_Status_CODE_BUILD_FAILED
+ status.StatusMessage = proto.String(s.GetStatus().GetMessage())
}
- file.Status = &pb.Status{Code: pb.Status_OK}
- if moduleToDeps[m.Name] != nil {
- file.Generated = moduleToDeps[m.Name].Generated
- file.Deps = moduleToDeps[m.Name].Deps
- continue
+ result := &pb.AnalysisResult{
+ SourceFilePath: s.GetPath(),
+ UnitId: s.GetPath(),
+ Status: status,
}
+ results = append(results, result)
- deps := transitiveDeps(m, javaModules)
var generated []*pb.GeneratedFile
- outPrefix := env.OutDir + "/"
- for _, d := range deps {
- if relPath, ok := strings.CutPrefix(d, outPrefix); ok {
- contents, err := os.ReadFile(d)
- if err != nil {
- fmt.Printf("Generated file %q not found - will be skipped.\n", d)
- continue
- }
+ for _, f := range s.Generated {
+ generated = append(generated, &pb.GeneratedFile{
+ Path: f.GetPath(),
+ Contents: f.GetContents(),
+ })
+ }
+ genUnit := &pb.BuildableUnit{
+ Id: "genfiles_for_" + s.GetPath(),
+ SourceFilePaths: s.GetDeps(),
+ GeneratedFiles: generated,
+ }
- generated = append(generated, &pb.GeneratedFile{
- Path: relPath,
- Contents: contents,
- })
+ unit := &pb.BuildableUnit{
+ Id: s.GetPath(),
+ Language: pb.Language_LANGUAGE_CPP,
+ SourceFilePaths: []string{s.GetPath()},
+ CompilerArguments: s.GetCompilerArguments(),
+ DependencyIds: []string{genUnit.GetId()},
+ }
+ units = append(units, unit, genUnit)
+ }
+ return results, units, nil
+}
+
+// findJavaModules tries to find the modules that cover the given file paths.
+// If a file is covered by multiple modules, the first module is returned.
+func findJavaModules(paths []string, modules map[string]*javaModule) map[string]string {
+ ret := make(map[string]string)
+ for name, module := range modules {
+ if strings.HasSuffix(name, ".impl") {
+ continue
+ }
+
+ for i, p := range paths {
+ if slices.Contains(module.Srcs, p) {
+ ret[p] = name
+ paths = append(paths[:i], paths[i+1:]...)
+ break
}
}
- moduleToDeps[m.Name] = &depsAndGenerated{deps, generated}
- file.Generated = generated
- file.Deps = deps
+ if len(paths) == 0 {
+ break
+ }
}
- return &pb.IdeAnalysis{
- Sources: sources,
+ return ret
+}
+
+func getJavaInputs(env Env, modulesByPath map[string]string, modules map[string]*javaModule) ([]*pb.AnalysisResult, []*pb.BuildableUnit) {
+ var results []*pb.AnalysisResult
+ unitsById := make(map[string]*pb.BuildableUnit)
+ for p, moduleName := range modulesByPath {
+ r := &pb.AnalysisResult{
+ SourceFilePath: p,
+ }
+ results = append(results, r)
+
+ m := modules[moduleName]
+ if m == nil {
+ r.Status = &pb.AnalysisResult_Status{
+ Code: pb.AnalysisResult_Status_CODE_NOT_FOUND,
+ StatusMessage: proto.String("File not found in any module."),
+ }
+ continue
+ }
+
+ r.UnitId = moduleName
+ r.Status = &pb.AnalysisResult_Status{Code: pb.AnalysisResult_Status_CODE_OK}
+ if unitsById[r.UnitId] != nil {
+ // File is covered by an already created unit.
+ continue
+ }
+
+ u := &pb.BuildableUnit{
+ Id: moduleName,
+ Language: pb.Language_LANGUAGE_JAVA,
+ SourceFilePaths: m.Srcs,
+ }
+ unitsById[u.Id] = u
+
+ q := list.New()
+ for _, d := range m.Deps {
+ q.PushBack(d)
+ }
+ for q.Len() > 0 {
+ name := q.Remove(q.Front()).(string)
+ mod := modules[name]
+ if mod == nil || unitsById[name] != nil {
+ continue
+ }
+
+ var paths []string
+ paths = append(paths, mod.Srcs...)
+ paths = append(paths, mod.SrcJars...)
+ paths = append(paths, mod.Jars...)
+ unitsById[name] = &pb.BuildableUnit{
+ Id: name,
+ SourceFilePaths: mod.Srcs,
+ GeneratedFiles: genFiles(env, paths),
+ }
+
+ for _, d := range mod.Deps {
+ q.PushBack(d)
+ }
+ }
}
+
+ units := make([]*pb.BuildableUnit, 0, len(unitsById))
+ for _, u := range unitsById {
+ units = append(units, u)
+ }
+ return results, units
+}
+
+// genFiles returns the generated files (paths that start with outDir/) for the
+// given paths. Generated files that do not exist are ignored.
+func genFiles(env Env, paths []string) []*pb.GeneratedFile {
+ prefix := env.OutDir + "/"
+ var ret []*pb.GeneratedFile
+ for _, p := range paths {
+ relPath, ok := strings.CutPrefix(p, prefix)
+ if !ok {
+ continue
+ }
+
+ contents, err := os.ReadFile(path.Join(env.RepoDir, p))
+ if err != nil {
+ continue
+ }
+
+ ret = append(ret, &pb.GeneratedFile{
+ Path: relPath,
+ Contents: contents,
+ })
+ }
+ return ret
}
// runMake runs Soong build for the given modules.
@@ -308,6 +411,7 @@
"TARGET_PRODUCT=" + env.LunchTarget.Product,
"TARGET_RELEASE=" + env.LunchTarget.Release,
"TARGET_BUILD_VARIANT=" + env.LunchTarget.Variant,
+ "TARGET_BUILD_TYPE=release",
"-k",
}
args = append(args, modules...)
@@ -319,7 +423,6 @@
}
type javaModule struct {
- Name string
Path []string `json:"path,omitempty"`
Deps []string `json:"dependencies,omitempty"`
Srcs []string `json:"srcs,omitempty"`
@@ -327,66 +430,23 @@
SrcJars []string `json:"srcjars,omitempty"`
}
-func loadJavaModules(env *Env) (map[string]*javaModule, map[string]*javaModule, error) {
+func loadJavaModules(env Env) (map[string]*javaModule, error) {
javaDepsPath := path.Join(env.RepoDir, env.OutDir, "soong/module_bp_java_deps.json")
data, err := os.ReadFile(javaDepsPath)
if err != nil {
- return nil, nil, err
+ return nil, err
}
- var moduleMapping map[string]*javaModule // module name -> module
- if err = json.Unmarshal(data, &moduleMapping); err != nil {
- return nil, nil, err
+ var ret map[string]*javaModule // module name -> module
+ if err = json.Unmarshal(data, &ret); err != nil {
+ return nil, err
}
- javaModules := make(map[string]*javaModule)
- javaFileToModuleMap := make(map[string]*javaModule)
- for name, module := range moduleMapping {
- if strings.HasSuffix(name, "-jarjar") || strings.HasSuffix(name, ".impl") {
- continue
- }
- module.Name = name
- javaModules[name] = module
- for _, src := range module.Srcs {
- if !slices.Contains(env.JavaFiles, src) {
- // We are only interested in active files.
- continue
- }
- if javaFileToModuleMap[src] != nil {
- // TODO(michaelmerg): Handle the case where a file is covered by multiple modules.
- log.Printf("File %q found in module %q but is already covered by module %q", src, module.Name, javaFileToModuleMap[src].Name)
- continue
- }
- javaFileToModuleMap[src] = module
+ // Add top level java_sdk_library for .impl modules.
+ for name, module := range ret {
+ if striped := strings.TrimSuffix(name, ".impl"); striped != name {
+ ret[striped] = module
}
}
- return javaModules, javaFileToModuleMap, nil
-}
-
-func transitiveDeps(m *javaModule, modules map[string]*javaModule) []string {
- var ret []string
- q := list.New()
- q.PushBack(m.Name)
- seen := make(map[string]bool) // module names -> true
- for q.Len() > 0 {
- name := q.Remove(q.Front()).(string)
- mod := modules[name]
- if mod == nil {
- continue
- }
-
- ret = append(ret, mod.Srcs...)
- ret = append(ret, mod.SrcJars...)
- ret = append(ret, mod.Jars...)
- for _, d := range mod.Deps {
- if seen[d] {
- continue
- }
- seen[d] = true
- q.PushBack(d)
- }
- }
- slices.Sort(ret)
- ret = slices.Compact(ret)
- return ret
+ return ret, nil
}