Merge "Add .hjar output file tag"
diff --git a/android/api_domain.go b/android/api_domain.go
index 0993e3d..3265148 100644
--- a/android/api_domain.go
+++ b/android/api_domain.go
@@ -70,13 +70,11 @@
return m
}
+// Do not create any dependency edges in Soong for now to skip visibility checks for some systemapi libraries.
+// Currently, all api_domain modules reside in build/orchestrator/apis/Android.bp
+// However, cc libraries like libsigchain (com.android.art) restrict their visibility to art/*
+// When the api_domain module types are collocated with their contributions, this dependency edge can be restored
func (a *apiDomain) DepsMutator(ctx BottomUpMutatorContext) {
- for _, cc := range a.properties.Cc_api_contributions {
- // Use FarVariationDependencies since the variants of api_domain is a subset of the variants of the dependency cc module
- // Creating a dependency on the first variant that matches (os,arch) is ok since this is a no-op in Soong
- // The primary function of this dependency is to create a connected graph in the corresponding bp2build workspace
- ctx.AddFarVariationDependencies(ctx.Target().Variations(), nil, cc)
- }
}
// API domain does not have any builld actions yet
diff --git a/android/soongconfig/modules.go b/android/soongconfig/modules.go
index 7d21b75..1519f60 100644
--- a/android/soongconfig/modules.go
+++ b/android/soongconfig/modules.go
@@ -642,9 +642,13 @@
// Extracts an interface from values containing the properties to apply based on config.
// If config does not match a value with a non-nil property set, the default value will be returned.
func (s *stringVariable) PropertiesToApply(config SoongConfig, values reflect.Value) (interface{}, error) {
+ configValue := config.String(s.variable)
+ if configValue != "" && !InList(configValue, s.values) {
+ return nil, fmt.Errorf("Soong config property %q must be one of %v, found %q", s.variable, s.values, configValue)
+ }
for j, v := range s.values {
f := values.Field(j)
- if config.String(s.variable) == v && !f.Elem().IsNil() {
+ if configValue == v && !f.Elem().IsNil() {
return f.Interface(), nil
}
}
@@ -861,3 +865,13 @@
}
var emptyInterfaceType = reflect.TypeOf(emptyInterfaceStruct{}).Field(0).Type
+
+// InList checks if the string belongs to the list
+func InList(s string, list []string) bool {
+ for _, s2 := range list {
+ if s2 == s {
+ return true
+ }
+ }
+ return false
+}
diff --git a/android/soongconfig/modules_test.go b/android/soongconfig/modules_test.go
index a7800e8..d5d87ef 100644
--- a/android/soongconfig/modules_test.go
+++ b/android/soongconfig/modules_test.go
@@ -303,6 +303,10 @@
Bool_var interface{}
}
+type stringSoongConfigVars struct {
+ String_var interface{}
+}
+
func Test_PropertiesToApply(t *testing.T) {
mt, _ := newModuleType(&ModuleTypeProperties{
Module_type: "foo",
@@ -365,6 +369,51 @@
}
}
+func Test_PropertiesToApply_String_Error(t *testing.T) {
+ mt, _ := newModuleType(&ModuleTypeProperties{
+ Module_type: "foo",
+ Config_namespace: "bar",
+ Variables: []string{"string_var"},
+ Properties: []string{"a", "b"},
+ })
+ mt.Variables = append(mt.Variables, &stringVariable{
+ baseVariable: baseVariable{
+ variable: "string_var",
+ },
+ values: []string{"a", "b", "c"},
+ })
+ stringVarPositive := &properties{
+ A: proptools.StringPtr("A"),
+ B: true,
+ }
+ conditionsDefault := &properties{
+ A: proptools.StringPtr("default"),
+ B: false,
+ }
+ actualProps := &struct {
+ Soong_config_variables stringSoongConfigVars
+ }{
+ Soong_config_variables: stringSoongConfigVars{
+ String_var: &boolVarProps{
+ A: stringVarPositive.A,
+ B: stringVarPositive.B,
+ Conditions_default: conditionsDefault,
+ },
+ },
+ }
+ props := reflect.ValueOf(actualProps)
+
+ _, err := PropertiesToApply(mt, props, Config(map[string]string{
+ "string_var": "x",
+ }))
+ expected := `Soong config property "string_var" must be one of [a b c], found "x"`
+ if err == nil {
+ t.Fatalf("Expected an error, got nil")
+ } else if err.Error() != expected {
+ t.Fatalf("Error message was not correct, expected %q, got %q", expected, err.Error())
+ }
+}
+
func Test_Bp2BuildSoongConfigDefinitions(t *testing.T) {
testCases := []struct {
desc string
diff --git a/apex/prebuilt.go b/apex/prebuilt.go
index 70308c8..39446a1 100644
--- a/apex/prebuilt.go
+++ b/apex/prebuilt.go
@@ -532,6 +532,10 @@
src = String(p.Arch.Arm64.Src)
case android.Riscv64:
src = String(p.Arch.Riscv64.Src)
+ // HACK: fall back to arm64 prebuilts, the riscv64 ones don't exist yet.
+ if src == "" {
+ src = String(p.Arch.Arm64.Src)
+ }
case android.X86:
src = String(p.Arch.X86.Src)
case android.X86_64:
diff --git a/bp2build/java_binary_host_conversion_test.go b/bp2build/java_binary_host_conversion_test.go
index c860844..e8551e5 100644
--- a/bp2build/java_binary_host_conversion_test.go
+++ b/bp2build/java_binary_host_conversion_test.go
@@ -33,7 +33,7 @@
}, tc)
}
-var fs = map[string]string{
+var testFs = map[string]string{
"test.mf": "Main-Class: com.android.test.MainClass",
"other/Android.bp": `cc_library_host_shared {
name: "jni-lib-1",
@@ -44,7 +44,7 @@
func TestJavaBinaryHost(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, exclude_srcs, jni_libs, javacflags, and manifest.",
- Filesystem: fs,
+ Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-1",
srcs: ["a.java", "b.java"],
@@ -77,7 +77,7 @@
func TestJavaBinaryHostRuntimeDeps(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, exclude_srcs, jni_libs, javacflags, and manifest.",
- Filesystem: fs,
+ Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-1",
static_libs: ["java-dep-1"],
@@ -107,7 +107,7 @@
func TestJavaBinaryHostLibs(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, libs.",
- Filesystem: fs,
+ Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-libs",
libs: ["java-lib-dep-1"],
diff --git a/bp2build/symlink_forest.go b/bp2build/symlink_forest.go
index 45817e3..81ec7ee 100644
--- a/bp2build/symlink_forest.go
+++ b/bp2build/symlink_forest.go
@@ -15,7 +15,9 @@
package bp2build
import (
+ "errors"
"fmt"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -49,6 +51,59 @@
okay atomic.Bool // Whether the forest was successfully constructed
}
+// A simple thread pool to limit concurrency on system calls.
+// Necessary because Go spawns a new OS-level thread for each blocking system
+// call. This means that if syscalls are too slow and there are too many of
+// them, the hard limit on OS-level threads can be exhausted.
+type syscallPool struct {
+ shutdownCh []chan<- struct{}
+ workCh chan syscall
+}
+
+type syscall struct {
+ work func()
+ done chan<- struct{}
+}
+
+func createSyscallPool(count int) *syscallPool {
+ result := &syscallPool{
+ shutdownCh: make([]chan<- struct{}, count),
+ workCh: make(chan syscall),
+ }
+
+ for i := 0; i < count; i++ {
+ shutdownCh := make(chan struct{})
+ result.shutdownCh[i] = shutdownCh
+ go result.worker(shutdownCh)
+ }
+
+ return result
+}
+
+func (p *syscallPool) do(work func()) {
+ doneCh := make(chan struct{})
+ p.workCh <- syscall{work, doneCh}
+ <-doneCh
+}
+
+func (p *syscallPool) shutdown() {
+ for _, ch := range p.shutdownCh {
+ ch <- struct{}{} // Blocks until the value is received
+ }
+}
+
+func (p *syscallPool) worker(shutdownCh <-chan struct{}) {
+ for {
+ select {
+ case <-shutdownCh:
+ return
+ case work := <-p.workCh:
+ work.work()
+ work.done <- struct{}{}
+ }
+ }
+}
+
// Ensures that the node for the given path exists in the tree and returns it.
func ensureNodeExists(root *instructionsNode, path string) *instructionsNode {
if path == "" {
@@ -317,6 +372,51 @@
}
}
+func removeParallelRecursive(pool *syscallPool, path string, fi os.FileInfo, wg *sync.WaitGroup) {
+ defer wg.Done()
+
+ if fi.IsDir() {
+ children := readdirToMap(path)
+ childrenWg := &sync.WaitGroup{}
+ childrenWg.Add(len(children))
+
+ for child, childFi := range children {
+ go removeParallelRecursive(pool, shared.JoinPath(path, child), childFi, childrenWg)
+ }
+
+ childrenWg.Wait()
+ }
+
+ pool.do(func() {
+ if err := os.Remove(path); err != nil {
+ fmt.Fprintf(os.Stderr, "Cannot unlink '%s': %s\n", path, err)
+ os.Exit(1)
+ }
+ })
+}
+
+func removeParallel(path string) {
+ fi, err := os.Lstat(path)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ return
+ }
+
+ fmt.Fprintf(os.Stderr, "Cannot lstat '%s': %s\n", path, err)
+ os.Exit(1)
+ }
+
+ wg := &sync.WaitGroup{}
+ wg.Add(1)
+
+ // Random guess as to the best number of syscalls to run in parallel
+ pool := createSyscallPool(100)
+ removeParallelRecursive(pool, path, fi, wg)
+ pool.shutdown()
+
+ wg.Wait()
+}
+
// Creates a symlink forest by merging the directory tree at "buildFiles" and
// "srcDir" while excluding paths listed in "exclude". Returns the set of paths
// under srcDir on which readdir() had to be called to produce the symlink
@@ -330,7 +430,7 @@
context.okay.Store(true)
- os.RemoveAll(shared.JoinPath(topdir, forest))
+ removeParallel(shared.JoinPath(topdir, forest))
instructions := instructionsFromExcludePathList(exclude)
go func() {
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 1e356e5..0669f65 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -91,7 +91,7 @@
flag.StringVar(&cmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
flag.BoolVar(&cmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode", false, "use bazel for analysis of certain modules")
- flag.BoolVar(&cmdlineArgs.BazelMode, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
+ flag.BoolVar(&cmdlineArgs.BazelModeStaging, "bazel-mode-staging", false, "use bazel for analysis of certain near-ready modules")
flag.BoolVar(&cmdlineArgs.BazelModeDev, "bazel-mode-dev", false, "use bazel for analysis of a large number of modules (less stable)")
// Flags that probably shouldn't be flags of soong_build but we haven't found
@@ -244,7 +244,7 @@
ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...)
// Create soong_injection repository
- soongInjectionFiles := bp2build.CreateSoongInjectionFiles(configuration, bp2build.CodegenMetrics{})
+ soongInjectionFiles := bp2build.CreateSoongInjectionFiles(configuration, bp2build.CreateCodegenMetrics())
absoluteSoongInjectionDir := shared.JoinPath(topDir, configuration.SoongOutDir(), bazel.SoongInjectionDirName)
for _, file := range soongInjectionFiles {
writeReadOnlyFile(absoluteSoongInjectionDir, file)
diff --git a/java/app_set.go b/java/app_set.go
index d99fadb..d8c2a8d 100644
--- a/java/app_set.go
+++ b/java/app_set.go
@@ -90,9 +90,10 @@
}
var TargetCpuAbi = map[string]string{
- "arm": "ARMEABI_V7A",
- "arm64": "ARM64_V8A",
- "riscv64": "RISCV64",
+ "arm": "ARMEABI_V7A",
+ "arm64": "ARM64_V8A",
+ // TODO: use "RISCV64" when that is supported in bundles
+ "riscv64": "ARM64_V8A",
"x86": "X86",
"x86_64": "X86_64",
}
diff --git a/java/java.go b/java/java.go
index 56a5b38..b6fc6b8 100644
--- a/java/java.go
+++ b/java/java.go
@@ -2344,7 +2344,7 @@
ctx.CreateBazelTargetModule(
bazel.BazelTargetModuleProperties{
Rule_class: "event_log_tags",
- Bzl_load_location: "//build/make/tools:event_log_tags.bzl",
+ Bzl_load_location: "//build/bazel/rules/java:event_log_tags.bzl",
},
android.CommonAttributes{Name: logtagsLibName},
&eventLogTagsAttributes{
diff --git a/tests/bootstrap_test.sh b/tests/bootstrap_test.sh
index 2331eb1..ba5ba8b 100755
--- a/tests/bootstrap_test.sh
+++ b/tests/bootstrap_test.sh
@@ -547,7 +547,6 @@
function test_bp2build_generates_marker_file {
setup
- create_mock_bazel
run_soong bp2build
diff --git a/tests/bp2build_bazel_test.sh b/tests/bp2build_bazel_test.sh
index ad21d7d..679ac55 100755
--- a/tests/bp2build_bazel_test.sh
+++ b/tests/bp2build_bazel_test.sh
@@ -50,7 +50,6 @@
function test_different_relative_outdir {
setup
- create_mock_bazel
mkdir -p a
touch a/g.txt
@@ -73,7 +72,6 @@
function test_different_absolute_outdir {
setup
- create_mock_bazel
mkdir -p a
touch a/g.txt
@@ -96,7 +94,6 @@
function test_bp2build_generates_all_buildfiles {
setup
- create_mock_bazel
mkdir -p foo/convertible_soong_module
cat > foo/convertible_soong_module/Android.bp <<'EOF'
@@ -167,7 +164,6 @@
function test_cc_correctness {
setup
- create_mock_bazel
mkdir -p a
cat > a/Android.bp <<EOF
diff --git a/tests/lib.sh b/tests/lib.sh
index 006186a..e40f0ad 100644
--- a/tests/lib.sh
+++ b/tests/lib.sh
@@ -82,10 +82,10 @@
}
function create_mock_soong {
+ create_mock_bazel
copy_directory build/blueprint
copy_directory build/soong
copy_directory build/make/tools/rbcrun
- copy_directory prebuilts/bazel/common/proto
symlink_directory prebuilts/sdk
symlink_directory prebuilts/go
diff --git a/tests/mixed_mode_test.sh b/tests/mixed_mode_test.sh
index 076ec4b..8949b42 100755
--- a/tests/mixed_mode_test.sh
+++ b/tests/mixed_mode_test.sh
@@ -12,7 +12,6 @@
function test_bazel_smoke {
setup
- create_mock_bazel
run_soong bp2build
@@ -21,7 +20,6 @@
function test_add_irrelevant_file {
setup
- create_mock_bazel
mkdir -p soong_tests/a/b
touch soong_tests/a/b/c.txt
@@ -33,7 +31,7 @@
}
EOF
- run_soong --bazel-mode nothing
+ run_soong --bazel-mode-staging nothing
if [[ ! -e out/soong/bp2build/soong_tests/a/b/BUILD.bazel ]]; then
fail "BUILD.bazel not created"
@@ -48,7 +46,7 @@
touch soong_tests/a/irrelevant.txt
- run_soong --bazel-mode nothing
+ run_soong --bazel-mode-staging nothing
local mtime_build2=$(stat -c "%y" out/soong/bp2build/soong_tests/a/b/BUILD.bazel)
local mtime_ninja2=$(stat -c "%y" out/soong/build.ninja)
diff --git a/ui/build/config.go b/ui/build/config.go
index 36119f0..de10112 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -236,6 +236,19 @@
return nil
}
+func defaultBazelProdMode(cfg *configImpl) bool {
+ // Envirnoment flag to disable Bazel for users which experience
+ // broken bazel-handled builds, or significant performance regressions.
+ if cfg.IsBazelMixedBuildForceDisabled() {
+ return false
+ }
+ // Darwin-host builds are currently untested with Bazel.
+ if runtime.GOOS == "darwin" {
+ return false
+ }
+ return true
+}
+
func NewConfig(ctx Context, args ...string) Config {
ret := &configImpl{
environ: OsEnvironment(),
@@ -774,6 +787,9 @@
c.arguments = append(c.arguments, arg)
}
}
+ if (!c.bazelProdMode) && (!c.bazelDevMode) && (!c.bazelStagingMode) {
+ c.bazelProdMode = defaultBazelProdMode(c)
+ }
}
func (c *configImpl) configureLocale(ctx Context) {
diff --git a/ui/build/config_test.go b/ui/build/config_test.go
index 968544b..940d85c 100644
--- a/ui/build/config_test.go
+++ b/ui/build/config_test.go
@@ -90,7 +90,9 @@
t.Fatal(err)
})
+ env := Environment([]string{})
c := &configImpl{
+ environ: &env,
parallel: -1,
keepGoing: -1,
}
diff --git a/ui/build/soong.go b/ui/build/soong.go
index ebf7166..4aded17 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -532,17 +532,22 @@
targets = append(targets, config.SoongNinjaFile())
}
- ninja("bootstrap", "bootstrap.ninja", targets...)
-
if shouldCollectBuildSoongMetrics(config) {
- soongBuildMetrics := loadSoongBuildMetrics(ctx, config)
- if soongBuildMetrics != nil {
- logSoongBuildMetrics(ctx, soongBuildMetrics)
- if ctx.Metrics != nil {
- ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
- }
+ soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb")
+ if err := os.Remove(soongBuildMetricsFile); err != nil && !os.IsNotExist(err) {
+ ctx.Verbosef("Failed to remove %s", soongBuildMetricsFile)
}
+ defer func() {
+ soongBuildMetrics := loadSoongBuildMetrics(ctx, soongBuildMetricsFile)
+ if soongBuildMetrics != nil {
+ logSoongBuildMetrics(ctx, soongBuildMetrics)
+ if ctx.Metrics != nil {
+ ctx.Metrics.SetSoongBuildMetrics(soongBuildMetrics)
+ }
+ }
+ }()
}
+ ninja("bootstrap", "bootstrap.ninja", targets...)
distGzipFile(ctx, config, config.SoongNinjaFile(), "soong")
distFile(ctx, config, config.SoongVarsFile(), "soong")
@@ -581,8 +586,7 @@
return config.SoongBuildInvocationNeeded()
}
-func loadSoongBuildMetrics(ctx Context, config Config) *soong_metrics_proto.SoongBuildMetrics {
- soongBuildMetricsFile := filepath.Join(config.LogsDir(), "soong_build_metrics.pb")
+func loadSoongBuildMetrics(ctx Context, soongBuildMetricsFile string) *soong_metrics_proto.SoongBuildMetrics {
buf, err := os.ReadFile(soongBuildMetricsFile)
if errors.Is(err, fs.ErrNotExist) {
// Soong may not have run during this invocation