Merge "Update bp2build denylist cause"
diff --git a/README.md b/README.md
index a67c393..b820fd1 100644
--- a/README.md
+++ b/README.md
@@ -561,27 +561,41 @@
## Developing for Soong
-To load Soong code in a Go-aware IDE, create a directory outside your android tree and then:
-```bash
-apt install bindfs
-export GOPATH=<path to the directory you created>
-build/soong/scripts/setup_go_workspace_for_soong.sh
-```
+To load the code of Soong in IntelliJ:
-This will bind mount the Soong source directories into the directory in the layout expected by
-the IDE.
-
+* File -> Open, open the `build/soong` directory. It will be opened as a new
+ project.
+* File -> Settings, then Languages & Frameworks -> Go -> GOROOT, then set it to
+ `prebuilts/go/linux-x86`
+* File -> Project Structure, then, Project Settings -> Modules, then Add
+ Content Root, then add the `build/blueprint` directory.
+* Optional: also add the `external/golang-protobuf` directory. In practice,
+ IntelliJ seems to work well enough without this, too.
### Running Soong in a debugger
-To run the soong_build process in a debugger, install `dlv` and then start the build with
-`SOONG_DELVE=<listen addr>` in the environment.
+To make `soong_build` wait for a debugger connection, install `dlv` and then
+start the build with `SOONG_DELVE=<listen addr>` in the environment.
For example:
```bash
-SOONG_DELVE=:1234 m nothing
+SOONG_DELVE=:5006 m nothing
```
-and then in another terminal:
+
+To make `soong_ui` wait for a debugger connection, use the `SOONG_UI_DELVE`
+variable:
+
```
-dlv connect :1234
+SOONG_UI_DELVE=:5006 m nothing
+```
+
+
+setting or unsetting `SOONG_DELVE` causes a recompilation of `soong_build`. This
+is because in order to debug the binary, it needs to be built with debug
+symbols.
+
+To test the debugger connection, run this command:
+
+```
+dlv connect :5006
```
If you see an error:
@@ -596,6 +610,21 @@
sudo sysctl -w kernel.yama.ptrace_scope=0
```
+To connect to the process using IntelliJ:
+
+* Run -> Edit Configurations...
+* Choose "Go Remote" on the left
+* Click on the "+" buttion on the top-left
+* Give it a nice name and set "Host" to localhost and "Port" to the port in the
+ environment variable
+
+Debugging works far worse than debugging Java, but is sometimes useful.
+
+Sometimes the `dlv` process hangs on connection. A symptom of this is `dlv`
+spinning a core or two. In that case, `kill -9` `dlv` and try again.
+Anecdotally, it _feels_ like waiting a minute after the start of `soong_build`
+helps.
+
## Contact
Email android-building@googlegroups.com (external) for any questions, or see
diff --git a/android/bazel.go b/android/bazel.go
index 2213945..87a9d50 100644
--- a/android/bazel.go
+++ b/android/bazel.go
@@ -308,8 +308,12 @@
// Per-module denylist to opt modules out of mixed builds. Such modules will
// still be generated via bp2build.
mixedBuildsDisabledList = []string{
- "libbrotli", // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy
- "minijail_constants_json", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
+ "libbrotli", // http://b/198585397, ld.lld: error: bionic/libc/arch-arm64/generic/bionic/memmove.S:95:(.text+0x10): relocation R_AARCH64_CONDBR19 out of range: -1404176 is not in [-1048576, 1048575]; references __memcpy
+ "func_to_syscall_nrs", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
+ "libseccomp_policy_app_zygote_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
+ "libseccomp_policy_app_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
+ "libseccomp_policy_system_sources", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
+ "minijail_constants_json", // http://b/200899432, bazel-built cc_genrule does not work in mixed build when it is a dependency of another soong module.
}
// Used for quicker lookups
diff --git a/bazel/aquery.go b/bazel/aquery.go
index 3ce86ce..0dedcf4 100644
--- a/bazel/aquery.go
+++ b/bazel/aquery.go
@@ -18,7 +18,6 @@
"encoding/json"
"fmt"
"path/filepath"
- "regexp"
"strings"
"github.com/google/blueprint/proptools"
@@ -60,8 +59,6 @@
InputDepSetIds []int
Mnemonic string
OutputIds []int
- TemplateContent string
- Substitutions []KeyValuePair
}
// actionGraphContainer contains relevant portions of Bazel's aquery proto, ActionGraphContainer.
@@ -103,14 +100,6 @@
artifactIdToPath map[int]string
}
-// The tokens should be substituted with the value specified here, instead of the
-// one returned in 'substitutions' of TemplateExpand action.
-var TemplateActionOverriddenTokens = map[string]string{
- // Uses "python3" for %python_binary% instead of the value returned by aquery
- // which is "py3wrapper.sh". See removePy3wrapperScript.
- "%python_binary%": "python3",
-}
-
func newAqueryHandler(aqueryResult actionGraphContainer) (*aqueryArtifactHandler, error) {
pathFragments := map[int]pathFragment{}
for _, pathFragment := range aqueryResult.PathFragments {
@@ -174,22 +163,7 @@
}
}
}
-
- // Filter out py3wrapper.sh & MANIFEST file. The middleman action returned by aquery
- // for python binary is the input list for a dependent of python binary, since py3wrapper.sh
- // and MANIFEST file could not be created in mixed build, they should be removed from
- // the input paths here.
- py3wrapper := "/py3wrapper.sh"
- manifestFile := regexp.MustCompile(".*/.+\\.runfiles/MANIFEST$")
- filteredInputPaths := []string{}
- for _, path := range inputPaths {
- if strings.HasSuffix(path, py3wrapper) || manifestFile.MatchString(path) {
- continue
- }
- filteredInputPaths = append(filteredInputPaths, path)
- }
-
- return filteredInputPaths, nil
+ return inputPaths, nil
}
func (a *aqueryArtifactHandler) artifactIdsFromDepsetId(depsetId int) ([]int, error) {
@@ -275,21 +249,6 @@
// Use hard links, because some soong actions expect real files (for example, `cp -d`).
buildStatement.Command = fmt.Sprintf("mkdir -p %[1]s && rm -f %[2]s && ln -f %[3]s %[2]s", outDir, out, in)
buildStatement.SymlinkPaths = outputPaths[:]
- } else if isTemplateExpandAction(actionEntry) && len(actionEntry.Arguments) < 1 {
- if len(outputPaths) != 1 {
- return nil, fmt.Errorf("Expect 1 output to template expand action, got: output %q", outputPaths)
- }
- expandedTemplateContent := expandTemplateContent(actionEntry)
- command := fmt.Sprintf(`echo "%[1]s" | sed "s/\\\\n/\\n/g" >> %[2]s && chmod a+x %[2]s`,
- escapeCommandlineArgument(expandedTemplateContent), outputPaths[0])
- buildStatement.Command = command
- } else if isPythonZipperAction(actionEntry) {
- if len(inputPaths) < 1 || len(outputPaths) != 1 {
- return nil, fmt.Errorf("Expect 1+ input and 1 output to python zipper action, got: input %q, output %q", inputPaths, outputPaths)
- }
- buildStatement.InputPaths, buildStatement.Command = removePy3wrapperScript(buildStatement)
- buildStatement.Command = addCommandForPyBinaryRunfilesDir(buildStatement, inputPaths[0], outputPaths[0])
- addPythonZipFileAsDependencyOfPythonBinary(&buildStatements, outputPaths[0])
} else if len(actionEntry.Arguments) < 1 {
return nil, fmt.Errorf("received action with no command: [%v]", buildStatement)
}
@@ -299,89 +258,10 @@
return buildStatements, nil
}
-// expandTemplateContent substitutes the tokens in a template.
-func expandTemplateContent(actionEntry action) string {
- replacerString := []string{}
- for _, pair := range actionEntry.Substitutions {
- value := pair.Value
- if val, ok := TemplateActionOverriddenTokens[pair.Key]; ok {
- value = val
- }
- replacerString = append(replacerString, pair.Key, value)
- }
- replacer := strings.NewReplacer(replacerString...)
- return replacer.Replace(actionEntry.TemplateContent)
-}
-
-func escapeCommandlineArgument(str string) string {
- // \->\\, $->\$, `->\`, "->\", \n->\\n
- replacer := strings.NewReplacer(
- `\`, `\\`,
- `$`, `\$`,
- "`", "\\`",
- `"`, `\"`,
- "\n", "\\n",
- )
- return replacer.Replace(str)
-}
-
-// removePy3wrapperScript removes py3wrapper.sh from the input paths and command of the action of
-// creating python zip file in mixed build. py3wrapper.sh is returned as input by aquery but
-// there is no action returned by aquery for creating it. So in mixed build "python3" is used
-// as the PYTHON_BINARY in python binary stub script, and py3wrapper.sh is not needed and should be
-// removed from input paths and command of creating python zip file.
-func removePy3wrapperScript(bs BuildStatement) (newInputPaths []string, newCommand string) {
- // Remove from inputs
- py3wrapper := "/py3wrapper.sh"
- filteredInputPaths := []string{}
- for _, path := range bs.InputPaths {
- if !strings.HasSuffix(path, py3wrapper) {
- filteredInputPaths = append(filteredInputPaths, path)
- }
- }
- newInputPaths = filteredInputPaths
-
- // Remove from command line
- var re = regexp.MustCompile(`\S*` + py3wrapper)
- newCommand = re.ReplaceAllString(bs.Command, "")
- return
-}
-
-// addCommandForPyBinaryRunfilesDir adds commands creating python binary runfiles directory
-// which currently could not be created with aquery output.
-func addCommandForPyBinaryRunfilesDir(bs BuildStatement, zipperCommandPath, zipFilePath string) string {
- // Unzip the zip file, zipFilePath looks like <python_binary>.zip
- runfilesDirName := zipFilePath[0:len(zipFilePath)-4] + ".runfiles"
- command := fmt.Sprintf("%s x %s -d %s", zipperCommandPath, zipFilePath, runfilesDirName)
- // Create a symblic link in <python_binary>.runfile/, which is the expected structure
- // when running the python binary stub script.
- command += fmt.Sprintf(" && ln -sf runfiles/__main__ %s", runfilesDirName)
- return bs.Command + " && " + command
-}
-
-// addPythonZipFileAsDependencyOfPythonBinary adds the action of generating python zip file as dependency of
-// the corresponding action of creating python binary stub script. In mixed build the dependent of python binary depends on
-// the action of createing python binary stub script only, which is not sufficient without the python zip file created.
-func addPythonZipFileAsDependencyOfPythonBinary(buildStatements *[]BuildStatement, pythonZipFilePath string) {
- for i, _ := range *buildStatements {
- if len((*buildStatements)[i].OutputPaths) >= 1 && (*buildStatements)[i].OutputPaths[0]+".zip" == pythonZipFilePath {
- (*buildStatements)[i].InputPaths = append((*buildStatements)[i].InputPaths, pythonZipFilePath)
- }
- }
-}
-
func isSymlinkAction(a action) bool {
return a.Mnemonic == "Symlink" || a.Mnemonic == "SolibSymlink"
}
-func isTemplateExpandAction(a action) bool {
- return a.Mnemonic == "TemplateExpand"
-}
-
-func isPythonZipperAction(a action) bool {
- return a.Mnemonic == "PythonZipper"
-}
-
func shouldSkipAction(a action) bool {
// TODO(b/180945121): Handle complex symlink actions.
if a.Mnemonic == "SymlinkTree" || a.Mnemonic == "SourceSymlinkManifest" {
diff --git a/bazel/aquery_test.go b/bazel/aquery_test.go
index f94d73f..88066c8 100644
--- a/bazel/aquery_test.go
+++ b/bazel/aquery_test.go
@@ -1015,326 +1015,6 @@
assertError(t, err, `Expect 1 input and 1 output to symlink action, got: input ["file"], output ["symlink" "other_symlink"]`)
}
-func TestTemplateExpandActionSubstitutions(t *testing.T) {
- const inputString = `
-{
- "artifacts": [{
- "id": 1,
- "pathFragmentId": 1
- }],
- "actions": [{
- "targetId": 1,
- "actionKey": "x",
- "mnemonic": "TemplateExpand",
- "configurationId": 1,
- "outputIds": [1],
- "primaryOutputId": 1,
- "executionPlatform": "//build/bazel/platforms:linux_x86_64",
- "templateContent": "Test template substitutions: %token1%, %python_binary%",
- "substitutions": [{
- "key": "%token1%",
- "value": "abcd"
- },{
- "key": "%python_binary%",
- "value": "python3"
- }]
- }],
- "pathFragments": [{
- "id": 1,
- "label": "template_file"
- }]
-}`
-
- actual, err := AqueryBuildStatements([]byte(inputString))
-
- if err != nil {
- t.Errorf("Unexpected error %q", err)
- }
-
- expectedBuildStatements := []BuildStatement{
- BuildStatement{
- Command: "echo \"Test template substitutions: abcd, python3\" | sed \"s/\\\\\\\\n/\\\\n/g\" >> template_file && " +
- "chmod a+x template_file",
- OutputPaths: []string{"template_file"},
- Mnemonic: "TemplateExpand",
- },
- }
- assertBuildStatements(t, expectedBuildStatements, actual)
-}
-
-func TestTemplateExpandActionNoOutput(t *testing.T) {
- const inputString = `
-{
- "artifacts": [{
- "id": 1,
- "pathFragmentId": 1
- }],
- "actions": [{
- "targetId": 1,
- "actionKey": "x",
- "mnemonic": "TemplateExpand",
- "configurationId": 1,
- "primaryOutputId": 1,
- "executionPlatform": "//build/bazel/platforms:linux_x86_64",
- "templateContent": "Test template substitutions: %token1%, %python_binary%",
- "substitutions": [{
- "key": "%token1%",
- "value": "abcd"
- },{
- "key": "%python_binary%",
- "value": "python3"
- }]
- }],
- "pathFragments": [{
- "id": 1,
- "label": "template_file"
- }]
-}`
-
- _, err := AqueryBuildStatements([]byte(inputString))
- assertError(t, err, `Expect 1 output to template expand action, got: output []`)
-}
-
-func TestPythonZipperActionSuccess(t *testing.T) {
- const inputString = `
-{
- "artifacts": [{
- "id": 1,
- "pathFragmentId": 1
- },{
- "id": 2,
- "pathFragmentId": 2
- },{
- "id": 3,
- "pathFragmentId": 3
- },{
- "id": 4,
- "pathFragmentId": 4
- },{
- "id": 5,
- "pathFragmentId": 10
- }],
- "actions": [{
- "targetId": 1,
- "actionKey": "x",
- "mnemonic": "PythonZipper",
- "configurationId": 1,
- "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
- "outputIds": [2],
- "inputDepSetIds": [1],
- "primaryOutputId": 2
- }],
- "depSetOfFiles": [{
- "id": 1,
- "directArtifactIds": [4, 3, 5]
- }],
- "pathFragments": [{
- "id": 1,
- "label": "python_binary"
- },{
- "id": 2,
- "label": "python_binary.zip"
- },{
- "id": 3,
- "label": "python_binary.py"
- },{
- "id": 9,
- "label": ".."
- }, {
- "id": 8,
- "label": "bazel_tools",
- "parentId": 9
- }, {
- "id": 7,
- "label": "tools",
- "parentId": 8
- }, {
- "id": 6,
- "label": "zip",
- "parentId": 7
- }, {
- "id": 5,
- "label": "zipper",
- "parentId": 6
- }, {
- "id": 4,
- "label": "zipper",
- "parentId": 5
- },{
- "id": 16,
- "label": "bazel-out"
- },{
- "id": 15,
- "label": "bazel_tools",
- "parentId": 16
- }, {
- "id": 14,
- "label": "k8-fastbuild",
- "parentId": 15
- }, {
- "id": 13,
- "label": "bin",
- "parentId": 14
- }, {
- "id": 12,
- "label": "tools",
- "parentId": 13
- }, {
- "id": 11,
- "label": "python",
- "parentId": 12
- }, {
- "id": 10,
- "label": "py3wrapper.sh",
- "parentId": 11
- }]
-}`
- actual, err := AqueryBuildStatements([]byte(inputString))
-
- if err != nil {
- t.Errorf("Unexpected error %q", err)
- }
-
- expectedBuildStatements := []BuildStatement{
- BuildStatement{
- Command: "../bazel_tools/tools/zip/zipper/zipper cC python_binary.zip __main__.py=bazel-out/k8-fastbuild/bin/python_binary.temp " +
- "__init__.py= runfiles/__main__/__init__.py= runfiles/__main__/python_binary.py=python_binary.py && " +
- "../bazel_tools/tools/zip/zipper/zipper x python_binary.zip -d python_binary.runfiles && ln -sf runfiles/__main__ python_binary.runfiles",
- InputPaths: []string{"../bazel_tools/tools/zip/zipper/zipper", "python_binary.py"},
- OutputPaths: []string{"python_binary.zip"},
- Mnemonic: "PythonZipper",
- },
- }
- assertBuildStatements(t, expectedBuildStatements, actual)
-}
-
-func TestPythonZipperActionNoInput(t *testing.T) {
- const inputString = `
-{
- "artifacts": [{
- "id": 1,
- "pathFragmentId": 1
- },{
- "id": 2,
- "pathFragmentId": 2
- }],
- "actions": [{
- "targetId": 1,
- "actionKey": "x",
- "mnemonic": "PythonZipper",
- "configurationId": 1,
- "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
- "outputIds": [2],
- "primaryOutputId": 2
- }],
- "pathFragments": [{
- "id": 1,
- "label": "python_binary"
- },{
- "id": 2,
- "label": "python_binary.zip"
- }]
-}`
- _, err := AqueryBuildStatements([]byte(inputString))
- assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input [], output ["python_binary.zip"]`)
-}
-
-func TestPythonZipperActionNoOutput(t *testing.T) {
- const inputString = `
-{
- "artifacts": [{
- "id": 1,
- "pathFragmentId": 1
- },{
- "id": 2,
- "pathFragmentId": 2
- },{
- "id": 3,
- "pathFragmentId": 3
- },{
- "id": 4,
- "pathFragmentId": 4
- },{
- "id": 5,
- "pathFragmentId": 10
- }],
- "actions": [{
- "targetId": 1,
- "actionKey": "x",
- "mnemonic": "PythonZipper",
- "configurationId": 1,
- "arguments": ["../bazel_tools/tools/zip/zipper/zipper", "cC", "python_binary.zip", "__main__.py\u003dbazel-out/k8-fastbuild/bin/python_binary.temp", "__init__.py\u003d", "runfiles/__main__/__init__.py\u003d", "runfiles/__main__/python_binary.py\u003dpython_binary.py", "runfiles/bazel_tools/tools/python/py3wrapper.sh\u003dbazel-out/bazel_tools/k8-fastbuild/bin/tools/python/py3wrapper.sh"],
- "inputDepSetIds": [1]
- }],
- "depSetOfFiles": [{
- "id": 1,
- "directArtifactIds": [4, 3, 5]
- }],
- "pathFragments": [{
- "id": 1,
- "label": "python_binary"
- },{
- "id": 2,
- "label": "python_binary.zip"
- },{
- "id": 3,
- "label": "python_binary.py"
- },{
- "id": 9,
- "label": ".."
- }, {
- "id": 8,
- "label": "bazel_tools",
- "parentId": 9
- }, {
- "id": 7,
- "label": "tools",
- "parentId": 8
- }, {
- "id": 6,
- "label": "zip",
- "parentId": 7
- }, {
- "id": 5,
- "label": "zipper",
- "parentId": 6
- }, {
- "id": 4,
- "label": "zipper",
- "parentId": 5
- },{
- "id": 16,
- "label": "bazel-out"
- },{
- "id": 15,
- "label": "bazel_tools",
- "parentId": 16
- }, {
- "id": 14,
- "label": "k8-fastbuild",
- "parentId": 15
- }, {
- "id": 13,
- "label": "bin",
- "parentId": 14
- }, {
- "id": 12,
- "label": "tools",
- "parentId": 13
- }, {
- "id": 11,
- "label": "python",
- "parentId": 12
- }, {
- "id": 10,
- "label": "py3wrapper.sh",
- "parentId": 11
- }]
-}`
- _, err := AqueryBuildStatements([]byte(inputString))
- assertError(t, err, `Expect 1+ input and 1 output to python zipper action, got: input ["../bazel_tools/tools/zip/zipper/zipper" "python_binary.py"], output []`)
-}
-
func assertError(t *testing.T, err error, expected string) {
t.Helper()
if err == nil {
diff --git a/python/binary.go b/python/binary.go
index 304c9a9..bf6167c 100644
--- a/python/binary.go
+++ b/python/binary.go
@@ -143,7 +143,7 @@
func NewBinary(hod android.HostOrDeviceSupported) (*Module, *binaryDecorator) {
module := newModule(hod, android.MultilibFirst)
- decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "", module)}
+ decorator := &binaryDecorator{pythonInstaller: NewPythonInstaller("bin", "")}
module.bootstrapper = decorator
module.installer = decorator
diff --git a/python/installer.go b/python/installer.go
index 515cc47..396f036 100644
--- a/python/installer.go
+++ b/python/installer.go
@@ -36,14 +36,12 @@
path android.InstallPath
androidMkSharedLibs []string
- module *Module
}
-func NewPythonInstaller(dir, dir64 string, module *Module) *pythonInstaller {
+func NewPythonInstaller(dir, dir64 string) *pythonInstaller {
return &pythonInstaller{
- dir: dir,
- dir64: dir64,
- module: module,
+ dir: dir,
+ dir64: dir64,
}
}
@@ -61,14 +59,7 @@
}
func (installer *pythonInstaller) install(ctx android.ModuleContext, file android.Path) {
- if ctx.ModuleType() == "python_binary_host" && installer.module.MixedBuildsEnabled(ctx) {
- label := installer.module.BazelModuleBase.GetBazelLabel(ctx, installer.module)
- binary, _ := ctx.Config().BazelContext.GetPythonBinary(label, android.GetConfigKey(ctx))
- bazelBinaryOutPath := android.PathForBazelOut(ctx, binary)
- installer.path = ctx.InstallFile(installer.installDir(ctx), bazelBinaryOutPath.Base(), bazelBinaryOutPath)
- } else {
- installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file)
- }
+ installer.path = ctx.InstallFile(installer.installDir(ctx), file.Base(), file)
}
func (installer *pythonInstaller) setAndroidMkSharedLibs(sharedLibs []string) {
diff --git a/python/test.go b/python/test.go
index 3cd900f..7413782 100644
--- a/python/test.go
+++ b/python/test.go
@@ -101,7 +101,7 @@
func NewTest(hod android.HostOrDeviceSupported) *Module {
module, binary := NewBinary(hod)
- binary.pythonInstaller = NewPythonInstaller("nativetest", "nativetest64", module)
+ binary.pythonInstaller = NewPythonInstaller("nativetest", "nativetest64")
test := &testDecorator{binaryDecorator: binary}
if hod == android.HostSupportedNoCross && test.testProperties.Test_options.Unit_test == nil {
diff --git a/ui/build/soong.go b/ui/build/soong.go
index a0f223b..1c7fbac 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -15,6 +15,7 @@
package build
import (
+ "fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -43,6 +44,12 @@
jsonModuleGraphTag = "modulegraph"
queryviewTag = "queryview"
soongDocsTag = "soong_docs"
+
+ // bootstrapEpoch is used to determine if an incremental build is incompatible with the current
+ // version of bootstrap and needs cleaning before continuing the build. Increment this for
+ // incompatible changes, for example when moving the location of the bpglob binary that is
+ // executed during bootstrap before the primary builder has had a chance to update the path.
+ bootstrapEpoch = 0
)
func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string) error {
@@ -121,20 +128,31 @@
}
}
-func writeEmptyGlobFile(ctx Context, path string) {
+func writeEmptyFile(ctx Context, path string) {
err := os.MkdirAll(filepath.Dir(path), 0777)
if err != nil {
- ctx.Fatalf("Failed to create parent directories of empty ninja glob file '%s': %s", path, err)
+ ctx.Fatalf("Failed to create parent directories of empty file '%s': %s", path, err)
}
- if _, err := os.Stat(path); os.IsNotExist(err) {
+ if exists, err := fileExists(path); err != nil {
+ ctx.Fatalf("Failed to check if file '%s' exists: %s", path, err)
+ } else if !exists {
err = ioutil.WriteFile(path, nil, 0666)
if err != nil {
- ctx.Fatalf("Failed to create empty ninja glob file '%s': %s", path, err)
+ ctx.Fatalf("Failed to create empty file '%s': %s", path, err)
}
}
}
+func fileExists(path string) (bool, error) {
+ if _, err := os.Stat(path); os.IsNotExist(err) {
+ return false, nil
+ } else if err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
func primaryBuilderInvocation(config Config, name string, output string, specificArgs []string) bootstrap.PrimaryBuilderInvocation {
commonArgs := make([]string, 0, 0)
@@ -166,10 +184,45 @@
}
}
+// bootstrapEpochCleanup deletes files used by bootstrap during incremental builds across
+// incompatible changes. Incompatible changes are marked by incrementing the bootstrapEpoch
+// constant. A tree is considered out of date for the current epoch of the
+// .soong.bootstrap.epoch.<epoch> file doesn't exist.
+func bootstrapEpochCleanup(ctx Context, config Config) {
+ epochFile := fmt.Sprintf(".soong.bootstrap.epoch.%d", bootstrapEpoch)
+ epochPath := filepath.Join(config.SoongOutDir(), epochFile)
+ if exists, err := fileExists(epochPath); err != nil {
+ ctx.Fatalf("failed to check if bootstrap epoch file %q exists: %q", epochPath, err)
+ } else if !exists {
+ // The tree is out of date for the current epoch, delete files used by bootstrap
+ // and force the primary builder to rerun.
+ os.Remove(filepath.Join(config.SoongOutDir(), "build.ninja"))
+ for _, globFile := range bootstrapGlobFileList(config) {
+ os.Remove(globFile)
+ }
+
+ // Mark the tree as up to date with the current epoch by writing the epoch marker file.
+ writeEmptyFile(ctx, epochPath)
+ }
+}
+
+func bootstrapGlobFileList(config Config) []string {
+ return []string{
+ config.NamedGlobFile(soongBuildTag),
+ config.NamedGlobFile(bp2buildTag),
+ config.NamedGlobFile(jsonModuleGraphTag),
+ config.NamedGlobFile(queryviewTag),
+ config.NamedGlobFile(soongDocsTag),
+ }
+}
+
func bootstrapBlueprint(ctx Context, config Config) {
ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
defer ctx.EndTrace()
+ // Clean up some files for incremental builds across incompatible changes.
+ bootstrapEpochCleanup(ctx, config)
+
mainSoongBuildExtraArgs := []string{"-o", config.SoongNinjaFile()}
if config.EmptyNinjaFile() {
mainSoongBuildExtraArgs = append(mainSoongBuildExtraArgs, "--empty-ninja-file")
@@ -232,8 +285,8 @@
// The glob .ninja files are subninja'd. However, they are generated during
// the build itself so we write an empty file if the file does not exist yet
// so that the subninja doesn't fail on clean builds
- for _, globFile := range globFiles {
- writeEmptyGlobFile(ctx, globFile)
+ for _, globFile := range bootstrapGlobFileList(config) {
+ writeEmptyFile(ctx, globFile)
}
var blueprintArgs bootstrap.Args
@@ -342,7 +395,7 @@
}
}()
- runMicrofactory(ctx, config, filepath.Join(config.HostToolDir(), "bpglob"), "github.com/google/blueprint/bootstrap/bpglob",
+ runMicrofactory(ctx, config, "bpglob", "github.com/google/blueprint/bootstrap/bpglob",
map[string]string{"github.com/google/blueprint": "build/blueprint"})
ninja := func(name, ninjaFile string, targets ...string) {