Add * support products and modules
Passing "*" to --products is equivalent to passing the all_named_producs
build variable
Passing "*" to --modules passes the contents of
PRODUCT_OUT/all_modules.txt
The total length of the text of all_modules can easily exceed the
maximum argument size for the OS. The proper solution is likely to call
getconf ARG_MAX, then concatenate the command to be executed to check
against the limit. Instead, the modules are processed in batches of 40k
modules. As long as module names don't get extremely long, this should
keep us under the ARG_MAX limit. In testing, using --modules "*" gets
split into two batches.
Test: build/make/tools/whichgit --modules "*"
Test: build/make/tools/whichgit --modules "*" --unused
Test: build/make/tools/whichgit --modules "*" --products "*"
Existing use-cases should remain unchanged:
TEST: build/make/tools/whichgit --modules framework
Change-Id: Ifa947daea2d439df0145e6def92637b67a8b5d22
diff --git a/tools/whichgit b/tools/whichgit
index 8cf84f5..55c8c6f 100755
--- a/tools/whichgit
+++ b/tools/whichgit
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
+import itertools
import os
import subprocess
import sys
@@ -10,15 +11,34 @@
check=True, capture_output=True, text=True).stdout.strip()
+def get_all_modules():
+ product_out = subprocess.run(["build/soong/soong_ui.bash", "--dumpvar-mode", "--abs", "PRODUCT_OUT"],
+ check=True, capture_output=True, text=True).stdout.strip()
+ result = subprocess.run(["cat", product_out + "/all_modules.txt"], check=True, capture_output=True, text=True)
+ return result.stdout.strip().split("\n")
+
+
+def batched(iterable, n):
+ # introduced in itertools 3.12, could delete once that's universally available
+ if n < 1:
+ raise ValueError('n must be at least one')
+ it = iter(iterable)
+ while batch := tuple(itertools.islice(it, n)):
+ yield batch
+
+
def get_sources(modules):
- result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f",
- "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja",
- "-t", "inputs", "-d", ] + modules,
- stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True)
- if result.returncode != 0:
- sys.stderr.write(result.stdout)
- sys.exit(1)
- return set([f for f in result.stdout.split("\n") if not f.startswith("out/")])
+ sources = set()
+ for module_group in batched(modules, 40_000):
+ result = subprocess.run(["./prebuilts/build-tools/linux-x86/bin/ninja", "-f",
+ "out/combined-" + os.environ["TARGET_PRODUCT"] + ".ninja",
+ "-t", "inputs", "-d", ] + list(module_group),
+ stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=False, text=True)
+ if result.returncode != 0:
+ sys.stderr.write(result.stdout)
+ sys.exit(1)
+ sources.update(set([f for f in result.stdout.split("\n") if not f.startswith("out/")]))
+ return sources
def m_nothing():
@@ -57,13 +77,13 @@
# Argument parsing
ap = argparse.ArgumentParser(description="List the required git projects for the given modules")
ap.add_argument("--products", nargs="*",
- help="The TARGET_PRODUCT to check. If not provided just uses whatever has"
- + " already been built")
+ help="One or more TARGET_PRODUCT to check, or \"*\" for all. If not provided"
+ + "just uses whatever has already been built")
ap.add_argument("--variants", nargs="*",
help="The TARGET_BUILD_VARIANTS to check. If not provided just uses whatever has"
+ " already been built, or eng if --products is supplied")
ap.add_argument("--modules", nargs="*",
- help="The build modules to check, or droid if not supplied")
+ help="The build modules to check, or \"*\" for all, or droid if not supplied")
ap.add_argument("--why", nargs="*",
help="Also print the input files used in these projects, or \"*\" for all")
ap.add_argument("--unused", help="List the unused git projects for the given modules rather than"
@@ -72,22 +92,33 @@
modules = args.modules if args.modules else ["droid"]
+ match args.products:
+ case ["*"]:
+ products = get_build_var("all_named_products").split(" ")
+ case _:
+ products = args.products
+
# Get the list of sources for all of the requested build combos
- if not args.products and not args.variants:
+ if not products and not args.variants:
+ m_nothing()
+ if args.modules == ["*"]:
+ modules = get_all_modules()
sources = get_sources(modules)
else:
- if not args.products:
+ if not products:
sys.stderr.write("Error: --products must be supplied if --variants is supplied")
sys.exit(1)
sources = set()
build_num = 1
- for product in args.products:
+ for product in products:
os.environ["TARGET_PRODUCT"] = product
variants = args.variants if args.variants else ["user", "userdebug", "eng"]
for variant in variants:
- sys.stderr.write(f"Analyzing build {build_num} of {len(args.products)*len(variants)}\r")
+ sys.stderr.write(f"Analyzing build {build_num} of {len(products)*len(variants)}\r")
os.environ["TARGET_BUILD_VARIANT"] = variant
m_nothing()
+ if args.modules == ["*"]:
+ modules = get_all_modules()
sources.update(get_sources(modules))
build_num += 1
sys.stderr.write("\n\n")