Make bazel build //external/... work by only allowing "approved" existing BUILD files in the Bazel workspace.
Test: bazel build //bionic/... //external/... //frameworks/... //system/...
Test: ./build/bazel/scripts/run_presubmits.sh
Change-Id: I91865ca87c6535053e7a14d2526ff3ce0991bfea
diff --git a/android/bazel.go b/android/bazel.go
index 9621f3e..ba29f6d 100644
--- a/android/bazel.go
+++ b/android/bazel.go
@@ -126,40 +126,21 @@
)
var (
- // Do not write BUILD files for these directories
- // NOTE: this is not recursive
- bp2buildDoNotWriteBuildFileList = []string{
- // Don't generate these BUILD files - because external BUILD files already exist
- "external/boringssl",
- "external/brotli",
- "external/dagger2",
- "external/flatbuffers",
- "external/gflags",
- "external/google-fruit",
- "external/grpc-grpc",
- "external/grpc-grpc/test/core/util",
- "external/grpc-grpc/test/cpp/common",
- "external/grpc-grpc/third_party/address_sorting",
- "external/nanopb-c",
- "external/nos/host/generic",
- "external/nos/host/generic/libnos",
- "external/nos/host/generic/libnos/generator",
- "external/nos/host/generic/libnos_datagram",
- "external/nos/host/generic/libnos_transport",
- "external/nos/host/generic/nugget/proto",
- "external/perfetto",
- "external/protobuf",
- "external/rust/cxx",
- "external/rust/cxx/demo",
- "external/ruy",
- "external/tensorflow",
- "external/tensorflow/tensorflow/lite",
- "external/tensorflow/tensorflow/lite/java",
- "external/tensorflow/tensorflow/lite/kernels",
- "external/tflite-support",
- "external/tinyalsa_new",
- "external/wycheproof",
- "external/libyuv",
+ // Keep any existing BUILD files (and do not generate new BUILD files) for these directories
+ bp2buildKeepExistingBuildFile = map[string]bool{
+ // This is actually build/bazel/build.BAZEL symlinked to ./BUILD
+ ".":/*recrusive = */ false,
+
+ "build/bazel":/* recursive = */ true,
+ "build/pesto":/* recursive = */ true,
+
+ // external/bazelbuild-rules_android/... is needed by mixed builds, otherwise mixed builds analysis fails
+ // e.g. ERROR: Analysis of target '@soong_injection//mixed_builds:buildroot' failed
+ "external/bazelbuild-rules_android":/* recursive = */ true,
+
+ "prebuilts/clang/host/linux-x86":/* recursive = */ false,
+ "prebuilts/sdk":/* recursive = */ false,
+ "prebuilts/sdk/tools":/* recursive = */ false,
}
// Configure modules in these directories to enable bp2build_available: true or false by default.
@@ -218,6 +199,7 @@
// libcxx
"libBionicBenchmarksUtils", // cc_library_static, fatal error: 'map' file not found, from libcxx
"fmtlib", // cc_library_static, fatal error: 'cassert' file not found, from libcxx
+ "fmtlib_ndk", // cc_library_static, fatal error: 'cassert' file not found
"libbase", // http://b/186826479, cc_library, fatal error: 'memory' file not found, from libcxx
// http://b/186024507: Includes errors because of the system_shared_libs default value.
@@ -229,6 +211,9 @@
"note_memtag_heap_async", // http://b/185127353: cc_library_static, error: feature.h not found
"note_memtag_heap_sync", // http://b/185127353: cc_library_static, error: feature.h not found
+ "libjemalloc5", // cc_library, ld.lld: error: undefined symbol: memset
+ "gwp_asan_crash_handler", // cc_library, ld.lld: error: undefined symbol: memset
+
// Tests. Handle later.
"libbionic_tests_headers_posix", // http://b/186024507, cc_library_static, sched.h, time.h not found
"libjemalloc5_integrationtest",
@@ -254,17 +239,12 @@
}
// Used for quicker lookups
- bp2buildDoNotWriteBuildFile = map[string]bool{}
bp2buildModuleDoNotConvert = map[string]bool{}
bp2buildCcLibraryStaticOnly = map[string]bool{}
mixedBuildsDisabled = map[string]bool{}
)
func init() {
- for _, moduleName := range bp2buildDoNotWriteBuildFileList {
- bp2buildDoNotWriteBuildFile[moduleName] = true
- }
-
for _, moduleName := range bp2buildModuleDoNotConvertList {
bp2buildModuleDoNotConvert[moduleName] = true
}
@@ -282,12 +262,21 @@
return bp2buildCcLibraryStaticOnly[ctx.Module().Name()]
}
-func ShouldWriteBuildFileForDir(dir string) bool {
- if _, ok := bp2buildDoNotWriteBuildFile[dir]; ok {
- return false
- } else {
+func ShouldKeepExistingBuildFileForDir(dir string) bool {
+ if _, ok := bp2buildKeepExistingBuildFile[dir]; ok {
+ // Exact dir match
return true
}
+ // Check if subtree match
+ for prefix, recursive := range bp2buildKeepExistingBuildFile {
+ if recursive {
+ if strings.HasPrefix(dir, prefix+"/") {
+ return true
+ }
+ }
+ }
+ // Default
+ return false
}
// MixedBuildsEnabled checks that a module is ready to be replaced by a
diff --git a/bp2build/conversion.go b/bp2build/conversion.go
index eb83b38..101ad3d 100644
--- a/bp2build/conversion.go
+++ b/bp2build/conversion.go
@@ -59,7 +59,7 @@
func createBuildFiles(buildToTargets map[string]BazelTargets, mode CodegenMode) []BazelFile {
files := make([]BazelFile, 0, len(buildToTargets))
for _, dir := range android.SortedStringKeys(buildToTargets) {
- if mode == Bp2Build && !android.ShouldWriteBuildFileForDir(dir) {
+ if mode == Bp2Build && android.ShouldKeepExistingBuildFileForDir(dir) {
fmt.Printf("[bp2build] Not writing generated BUILD file for dir: '%s'\n", dir)
continue
}
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 044689e..70c8856 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -17,6 +17,7 @@
import (
"flag"
"fmt"
+ "io/fs"
"io/ioutil"
"os"
"path/filepath"
@@ -357,6 +358,80 @@
}
}
+// Find BUILD files in the srcDir which...
+//
+// - are not on the allow list (android/bazel.go#ShouldKeepExistingBuildFileForDir())
+//
+// - won't be overwritten by corresponding bp2build generated files
+//
+// And return their paths so they can be left out of the Bazel workspace dir (i.e. ignored)
+func getPathsToIgnoredBuildFiles(topDir string, outDir string, generatedRoot string) ([]string, error) {
+ paths := make([]string, 0)
+
+ err := filepath.WalkDir(topDir, func(fFullPath string, fDirEntry fs.DirEntry, err error) error {
+ if err != nil {
+ // Warn about error, but continue trying to walk the directory tree
+ fmt.Fprintf(os.Stderr, "WARNING: Error accessing path '%s', err: %s\n", fFullPath, err)
+ return nil
+ }
+ if fDirEntry.IsDir() {
+ // Don't ignore entire directories
+ return nil
+ }
+ if !(fDirEntry.Name() == "BUILD" || fDirEntry.Name() == "BUILD.bazel") {
+ // Don't ignore this file - it is not a build file
+ return nil
+ }
+ f := strings.TrimPrefix(fFullPath, topDir+"/")
+ if strings.HasPrefix(f, ".repo/") {
+ // Don't check for files to ignore in the .repo dir (recursively)
+ return fs.SkipDir
+ }
+ if strings.HasPrefix(f, outDir+"/") {
+ // Don't check for files to ignore in the out dir (recursively)
+ return fs.SkipDir
+ }
+ if strings.HasPrefix(f, generatedRoot) {
+ // Don't check for files to ignore in the bp2build dir (recursively)
+ // NOTE: This is usually under outDir
+ return fs.SkipDir
+ }
+ fDir := filepath.Dir(f)
+ if android.ShouldKeepExistingBuildFileForDir(fDir) {
+ // Don't ignore this existing build file
+ return nil
+ }
+ f_bp2build := shared.JoinPath(topDir, generatedRoot, f)
+ if _, err := os.Stat(f_bp2build); err == nil {
+ // If bp2build generated an alternate BUILD file, don't exclude this workspace path
+ // BUILD file clash resolution happens later in the symlink forest creation
+ return nil
+ }
+ fmt.Fprintf(os.Stderr, "Ignoring existing BUILD file: %s\n", f)
+ paths = append(paths, f)
+ return nil
+ })
+
+ return paths, err
+}
+
+// Returns temporary symlink forest excludes necessary for bazel build //external/... (and bazel build //frameworks/...) to work
+func getTemporaryExcludes() []string {
+ excludes := make([]string, 0)
+
+ // FIXME: 'autotest_lib' is a symlink back to external/autotest, and this causes an infinite symlink expansion error for Bazel
+ excludes = append(excludes, "external/autotest/venv/autotest_lib")
+
+ // FIXME: The external/google-fruit/extras/bazel_root/third_party/fruit dir is poison
+ // It contains several symlinks back to real source dirs, and those source dirs contain BUILD files we want to ignore
+ excludes = append(excludes, "external/google-fruit/extras/bazel_root/third_party/fruit")
+
+ // FIXME: 'frameworks/compile/slang' has a filegroup error due to an escaping issue
+ excludes = append(excludes, "frameworks/compile/slang")
+
+ return excludes
+}
+
// Run Soong in the bp2build mode. This creates a standalone context that registers
// an alternate pipeline of mutators and singletons specifically for generating
// Bazel BUILD files instead of Ninja files.
@@ -415,6 +490,18 @@
excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir)
}
+ // FIXME: Don't hardcode this here
+ topLevelOutDir := "out"
+
+ pathsToIgnoredBuildFiles, err := getPathsToIgnoredBuildFiles(topDir, topLevelOutDir, generatedRoot)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error walking SrcDir: '%s': %s\n", configuration.SrcDir(), err)
+ os.Exit(1)
+ }
+ excludes = append(excludes, pathsToIgnoredBuildFiles...)
+
+ excludes = append(excludes, getTemporaryExcludes()...)
+
symlinkForestDeps := bp2build.PlantSymlinkForest(
topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes)